#define TESTSUITE
//#define TLDB
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using miew.Concurrency;
using miew.Tokenization;
using miew.BitArray;
using miew.String;
using miew.String.Builder;
using miew.Enumerable;
using miew.Debugging;
using miew.Math;
using agree;
//using tldb;
#pragma warning disable 0618, 0414
class MainClass
{
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
static void Main(string[] args)
{
Console.WriteLine("gcServer={0}, agree {1} sizeof(arr_tfs_entry)={2}",
System.Runtime.GCSettings.IsServerGC,
SysObj.build_type,
System.Runtime.InteropServices.Marshal.SizeOf(typeof(arr_tfs_entry)));
new MainClass()._Main(args);
//Console.WriteLine("{0} {1} {2}", UnificationNWay.newway, UnificationNWay.oldway, (double)UnificationNWay.newway / (UnificationNWay.newway + UnificationNWay.oldway));
//Console.WriteLine("{0}", ParseControl.max_alloc);
if (Debugger.IsAttached)
{
Console.WriteLine("done! press <enter> to exit");
Console.ReadLine();
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
public void ParseG6Morph(Grammar g)
{
int c_parses;
using (TimingReport tr = new TimingReport(Console.Out, "Parsing..."))
{
c_parses = 0;
try
{
using (ParseControl pc = DoParse(g, "give that cat to the dog"))
{
if (pc != null)
{
c_parses = pc.chart.CompletedEdges.Count();
Console.WriteLine(c_parses);
}
}
}
catch (AggregateException)
{
}
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
public void ParseThai(Grammar g)
{
int c_parses;
Tfs.MonospaceFormatter mf = null;
if (f_tfs)
mf = new Tfs.MonospaceFormatter(g.tm);
using (TimingReport tr = new TimingReport(Console.Out, "Parsing..."))
{
//String s_sent = "เขา ไป ซื้อ ดอกไม้ ที่ ตลาด ไป เยี่ยม เพื่อน และ";
if (sentences_to_parse.Count > 0)
{
int i = 0;
foreach (String l in sentences_to_parse)
{
c_parses = 0;
try
{
TimingReport tr2 = null;
if (f_trees || f_tfs || config.system.tw_debug != null)
tr2 = new TimingReport(Console.Out, l);
using (ParseControl pc = DoParse(g, l))
{
if (pc != null)
{
c_parses = pc.chart.CompletedEdges.Count();
if (f_chart_dump)
pc.chart.DumpChart(String.Format("chart_dump-{0}.html", i));
if (f_trees || f_tfs)
{
foreach (var px in pc.chart.AllDerivations)
{
if (f_trees)
Console.WriteLine(g.nl.GetParseTree(pc.chart, px));
if (f_tfs)
{
Console.WriteLine(mf.FormatEdge(px.UnpackedTfs));
}
}
}
}
if (f_trees)
tr2.Dispose();
}
}
catch (ParseException)
{
}
catch (AggregateException)
{
}
Console.WriteLine("{0} parses", c_parses);
i++;
}
}
else
{
foreach (String l in File.ReadAllLines("grammars/thai/testsuite.txt").Skip(config.system.submitter.skipitems).Take(config.system.submitter.takeitems))
{
String[] rgs = l.Split('\t');
int id = int.Parse(rgs[0]);
c_parses = 0;
try
{
TimingReport tr2 = null;
if (f_trees || f_tfs || f_item_info || config.system.tw_debug != null)
tr2 = new TimingReport(Console.Out, String.Format("{0}", id));
using (ParseControl pc = DoParse(g, rgs[1]))
{
if (pc != null)
{
c_parses = pc.chart.CompletedEdges.Count();
if (f_chart_dump)
pc.chart.DumpChart(String.Format("chart_dump-{0}.html", id));
if (f_trees || f_tfs)
{
foreach (var px in pc.chart.AllDerivations)
{
if (f_trees)
Console.WriteLine(g.nl.GetParseTree(pc.chart, px));
if (f_tfs)
{
Console.WriteLine(mf.FormatEdge(px.UnpackedTfs));
}
}
}
}
}
if (f_trees || f_tfs || f_item_info || config.system.tw_debug != null)
tr2.Dispose();
}
catch (ParseException)
{
}
catch (AggregateException)
{
}
int c_parses_exp = int.Parse(rgs[2]);
if (c_parses != c_parses_exp)
{
String msg = String.Format("parser regression test failed for [{0}]\n{1} parses, expected {2}", rgs[1], c_parses, c_parses_exp);
//throw new Exception(msg);
Console.WriteLine(msg);
}
#if FCTC_STATS
g.tm.TypePatternReport();
#endif
//LexicalEntry.ReleaseAll(g.lex.lex_entries);
//foreach (GrammarRule r in g._grammar_rules)
// r.ReleaseExpanded();
if (f_garbage)
Console.WriteLine(g.GarbageReport());
}
}
#if false
Console.WriteLine("processing {0} test items", igts.Count);
foreach (var testitem in igts)
{
Phrase phr = testitem.phr;
//phr = db.GetPhrase(219755);
String[] sent = phr.Parts.Select(e => e.FullDop.Thai(default(DbObj.LayoutOpts))).ToArray();
String s_sent = sent.StringJoin(" ");
Console.WriteLine();
Console.WriteLine("Parsing: P{0} {1}", phr.Id, s_sent);
try
{
using (ParseChart pc = DoParse(sent))
{
c_parses = pc.CompletedParses.Count;
}
catch (AggregateException ex)
{
c_parses = 0;
//Console.WriteLine(ex.Flatten().InnerException);
}
//Console.WriteLine();
//Console.WriteLine(g.GarbageReport());
Console.WriteLine("P{0} {1} Completed Parses: {2}", phr.Id, s_sent, c_parses);
//Console.ReadLine();
sw_parsecounts.WriteLine("{0}\t{1}\t{2}", phr.Id, s_sent, c_parses);
}
#endif
}
//LexicalEntry.ReleaseAll(g.lex.lex_entries);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ParseErg
{
int q = 0;
MainClass mc;
String[] original_lines;
Tfs.MonospaceFormatter mf = null;
String fdate;
Grammar g;
Dictionary<int, RegressInfo> hike_regress;
ItemInfo[] rg_items;
int gen_base;
int c_unif_tot = 0;
long ms_tot = 0;
struct RegressInfo
{
public int id;
public int readings;
public int cunif;
};
struct ItemInfo
{
public int id;
public bool skip;
public String text;
};
public ParseErg(MainClass mc)
{
this.mc = mc;
}
public void Parse(Grammar g)
{
this.g = g;
if (mc.f_tfs)
mf = new Tfs.MonospaceFormatter(g.tm);
fdate = String.Format("/programming/analytical-grammar/hike/{0:yyyyMMdd\\-hhmmss}-hike-results.txt", DateTime.Now);
File.Delete(fdate);
hike_regress = File.ReadAllLines("/programming/analytical-grammar/hike/agree-regress.txt").Select(oo => oo.Split('\t')).Select(vv =>
new RegressInfo { id = int.Parse(vv[0]), readings = vv[1] == "" ? 0 : int.Parse(vv[1]), cunif = vv[2] == "" ? 0 : int.Parse(vv[2]) }).ToDictionary(a => a.id);
original_lines = System.IO.File.ReadAllLines(@"/programming/analytical-grammar/hike/hike-results-use.txt", Encoding.UTF8)
.ToArray();
rg_items = original_lines
.Select(vvv =>
{
var wxyz = vvv.Split('\t');
var rsr = wxyz[0];
bool sk = rsr[0] == '#';
rsr = rsr.Trim('#');
return new ItemInfo
{
id = int.Parse(rsr),
skip = sk,
text = wxyz[1].Trim('\"'),
/* sec = (Double)int.Parse(wxyz[2]) / 1000,
readings = int.Parse(wxyz[3]) */
};
}).ToArray();
String[] rg_sent = rg_items.Skip(mc.config.system.submitter.skipitems).Take(mc.config.system.submitter.takeitems).Select(vsr => vsr.text).ToArray();
if (mc.sentences_to_parse.Count == 0)
mc.sentences_to_parse.AddRange(rg_sent);
if (mc.f_s100)
{
mc.sentences_to_parse.Clear();
mc.sentences_to_parse.AddRange(Enumerable.Repeat("Keep to the right in the track and yield one of the ski tracks if you meet someone.", 100));
}
//if (!Debugger.IsAttached)
//{
// using (var prelim = mc.DoParse(g, "This is a preliminary parse."))
// {
// }
//}
//File.Delete("hike-full-regress.txt");
//if (mc.config.system.submitter.TaskCount > 1)
// bar = new Barrier(0, b =>
// {
// GC.Collect();
// GC.WaitForPendingFinalizers();
// GC.Collect();
// });
using (TimingReport tr = new TimingReport(Console.Out, "Parsing..."))
{
bw = Console.BufferWidth - 1;
if (!mc.f_no_report)
{
Console.Out.WriteLineColorSync(hdr_a.PadLeft(bw));
Console.Out.WriteLineColorSync("$cyan " + hdr_b.PadLeft(bw));
}
gen_base = GC.CollectionCount(2);
if (mc.config.system.submitter.TaskCount == 1)
ErgParseTask();
else
{
Task[] rgtasks = new Task[mc.config.system.submitter.TaskCount - 1];
for (int i = 0; i < mc.config.system.submitter.TaskCount - 1; i++)
rgtasks[i] = Task.Factory.StartNew(ErgParseTask);
ErgParseTask();
Task.WaitAll(rgtasks);
}
}
File.WriteAllLines(fdate, g_results.OrderBy(ss => int.Parse(ss.Substring(0, ss.IndexOf('\t') + 1))));
}
const string hdr_a = "+------totals------+ +------morph-------+ +--------------parse--------------+ +----pack----+ --unpack- --sys--";
const string hdr_b = " read wall unif %t unif % lx ana %t QC unif % pas-e act-e cmp-e eq fwd rev %t unif mem GC";
const int MIN_TRIM = 5;
int bw;
List<String> g_results = new List<string>();
int c_tasks = 0;
//Barrier bar;
public static string _PCT(double pct)
{
if (double.IsInfinity(pct))
return "oo";
if (double.IsNaN(pct))
return " ";
if (pct < 1.0)
return ((int)(pct * 100)).ToString();
return "%%";
}
void ErgParseTask()
{
int tsk = Interlocked.Increment(ref c_tasks) - 1;
List<String> results = new List<string>();
int item_ix;
while ((item_ix = Interlocked.Increment(ref q) - 1) < mc.sentences_to_parse.Count)
{
String sent = mc.sentences_to_parse[item_ix];
//String disp_sent = sent.SubstringOrLess(0, 73);
double mem = 0;
int num;
ParseControl.Stats stats = null;
{
int ix;
if ((ix = Array.FindIndex(rg_items, oo => oo.text == sent)) != -1)
{
num = rg_items[ix].id;
if (rg_items[ix].skip)
goto skipit;
item_ix = ix;
}
else
{
num = item_ix;
}
}
StringBuilder report = new StringBuilder();
try
{
using (ParseControl ctrl = g.sub.Parse(sent).Result)
{
stats = ctrl.stats;
if (mc.f_quick_exit)
Environment.Exit(0);
if (ctrl == null)
{
report.AppendFormat("$red null chart");
report.AppendLine();
}
else
{
int c_deriv = ctrl.chart.AllDerivations.Count;
#if DERIVATION_LIST
foreach (var ez in ctrl.chart)
{
if (!ez.DerivedEdges.IsDistinct())
throw new Exception();
Console.WriteLine("{0}", ez);
foreach (var qqqqs in ez.DerivedEdges)
{
if (qqqqs is PassiveEdge.CompletedParse)
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine(" {0}", qqqqs);
Console.ResetColor();
}
Console.WriteLine();
}
#endif
if (mc.f_chart_dump)
ctrl.chart.DumpChart(String.Format("chart_dump-{0}.html", item_ix));
if (mc.f_trees || mc.f_tfs)
{
foreach (var px in ctrl.chart.AllDerivations)
{
if (mc.f_trees)
Console.WriteLine(g.nl.GetParseTree(ctrl.chart, px));
if (mc.f_tfs)
Console.WriteLine(mf.FormatEdge(px.UnpackedTfs));
}
}
#if false
//foreach (var tr in pc.chart.CompletedParses.SelectMany(cp=>cp.Derivations.OfType<ParseChart.ParseTree>()).OrderBy(z => z.UnpackTask.Result.TfsHash()))
foreach (var drv in pc.chart.CompletedParses.SelectMany(cp => cp.Derivations.OfType<ParseChart.ParseTree>()).OrderBy(z => z.DerivationHash))
{
Console.WriteLine("#{0:X} =========================", drv.DerivationHash);
Console.WriteLine(drv.TreeDisplay());
}
Console.WriteLine();
#endif
#if false
System.IO.File.WriteAllText(String.Format("001_{0:X}.tfs", up2[9].id), up2[9].ToPathList(false));
System.IO.File.WriteAllText(String.Format("002_{0:X}.tfs", up2[10].id), up2[9].ToPathList(false));
#endif
}
}
if (stats == null)
goto skipit;
//Console.Out.WriteLineColorSync("$red pausing...");
//Thread.Sleep(20000);
String problems = "";
var a = hike_regress.FirstOrDefault(kvp => kvp.Key == num).Value;
if (a.id == num)
{
int c_derivations = stats.Parsing.Unpacking.c_derivations;
if (num != 0 && a.readings != c_derivations)
{
problems += String.Format("error: $red derivations {0}$ , expected {1}", c_derivations, a.readings);
problems += Environment.NewLine;
}
else if (mc.c_derivations_expected != int.MaxValue && mc.c_derivations_expected != c_derivations)
{
problems += String.Format("error: $red derivations {0}$ , expected {1}", c_derivations, mc.c_derivations_expected);
problems += Environment.NewLine;
}
if (mc.f_unif_regress && a.cunif != stats.Totals.Unification.c_attempted)
problems += String.Format("error: unif {0}, expected {1}", stats.Totals.Unification.c_attempted, a.cunif);
}
if (!mc.f_no_report || problems != "")
{
//mem = (double)GC.GetTotalMemory(false) / (1024 * 1024 * 1024);
mem = (double)Process.GetCurrentProcess().PrivateMemorySize64 / (1024 * 1024 * 1024);
String gcinfo = String.Format("{0}", GC.CollectionCount(2) - gen_base);
int trim_val = Math.Max(MIN_TRIM, bw - 9 - hdr_a.Length);
String ds = sent.SubstringOrLess(0, trim_val);
report.AppendFormat("{0,3:##0} {1,4:##0} $yellow {2}", item_ix, num, ds);
report.Append(new String(' ', Math.Max(0, bw - 9 - hdr_a.Length - ds.Length)));
// read wall unif. %t unif. ok lx ana %t unif. QC pas act cmp eq fwd rev %t unif. GB GC
String rxt =
String.Format("$magenta {0,5:G}$ {1,6:#.###} $green {2,7:G}$ $darkred {3,2}$ $green {4,5:G}$ {5,2} {6,3:G} {7,4:G} $darkred {8,2}$ {9,2}$green {10,7:G}$ {11,2} {12,6:G} {13,6:G} {14,5:G}{15,5:G}{16,5:G}{17,5:G} $darkred {18,2}$ $green {19,6:G}$ {20:#.##} {21,2}",
stats.Parsing.Unpacking.c_derivations,
stats.Totals.sec_time,
stats.Totals.Unification.c_attempted,
_PCT(stats.Morphology.TimeRatio),
stats.Morphology.Unification.c_attempted,
_PCT(stats.Morphology.Unification.SuccessRatio),
stats.Morphology.c_lexical_transforms,
stats.Morphology.c_analysis_stacks,
_PCT(stats.Parsing.Chart.TimeRatio),
_PCT(stats.Parsing.QuickCheck.PercentAvoided),
stats.Parsing.Unification.c_attempted,
_PCT(stats.Parsing.Unification.SuccessRatio),
stats.Parsing.Chart.c_passive_edges,
stats.Parsing.Chart.c_active_edges,
stats.Parsing.Chart.c_root_edges,
stats.Parsing.Packing.c_equivalence,
stats.Parsing.Packing.c_proactive,
stats.Parsing.Packing.c_retroactive,
_PCT(stats.Parsing.Unpacking.TimeRatio),
stats.Parsing.Unpacking.Unification.c_attempted,
mem.ToString("#.##").TrimEndOrPadLeft(4),
gcinfo);
report.AppendLine(rxt);
report.AppendLine(problems);
}
//if (PassiveEdge.Derived.qc > 0)
//{
// Console.Write("{0} ", PassiveEdge.Derived.qc);
// PassiveEdge.Derived.qc = 0;
//}
//if (ctrl.chart.hs.Count > 0)
// report.AppendLine(" " + ctrl.chart.hs.StringJoin(" "));
//String fr = String.Format("{0}\t{1}\t{2}", num, c_parses, up2
// .Select(tz => tz.TfsHash())
// .OrderBy(tz => tz)
// .Select(tz => tz.ToString("X"))
// .StringJoin("\t"));
//File.AppendAllText("hike-full-regress.txt", fr + Environment.NewLine);
}
//Console.WriteLine(g.tm.GlbCacheInfo());
catch (Exception ex)
{
report.AppendFormat("{0,3:##0} {1,3:##0} $yellow {2} ", item_ix, num, sent);
if (ex is AggregateException)
ex = ((AggregateException)ex).Flatten().InnerExceptions.FirstOrDefault(x => !(x is AggregateException)) ?? ex;
report.AppendFormatLine("\r\n$red {0} {1}$ \r\n", ex.GetType().Name, ex.Message);
if (!(ex is ParseException))
{
report.AppendLine(ex.StackTrace);
lock (Console.Out)
{
Console.Out.WriteLineColorSync(report.ToString());
Console.ResetColor();
Console.Out.Flush();
}
throw ex;
}
}
#if FCTC_STATS
g.tm.TypePatternReport();
#endif
if (report.Length > 0)
{
if (mc.config.system.submitter.TaskCount > 1 && Monitor.TryEnter(Console.Out))
{
foreach (string lll in report.ToString().Split(new Char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries))
Console.Out.WriteLineColorSync(lll);
Monitor.Exit(Console.Out);
}
else Task.Factory.StartNew(() =>
{
Monitor.Enter(Console.Out);
foreach (string lll in report.ToString().Split(new Char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries))
Console.Out.WriteLineColorSync(lll);
Monitor.Exit(Console.Out);
});
}
//if (bar != null)
//{
// if (bar.ParticipantCount > 0)
// bar.SignalAndWait();
// else if (mem > 16.0)
// {
// bar.AddParticipants(c_tasks);
// bar.SignalAndWait();
// bar.RemoveParticipants(c_tasks);
// }
//}
skipit:
String s_c_parses;
String s_sec;
String s_c_unif;
String s_forest;
String s_pack_rej;
if (stats == null)
{
s_pack_rej = s_forest = s_sec = s_c_unif = s_c_parses = "".Quotes();
}
else
{
s_c_unif = stats.Totals.Unification.c_attempted.ToString();
s_sec = stats.Totals.sec_time.ToString();
s_c_parses = stats.Parsing.Unpacking.c_derivations.ToString();
s_forest = stats.Parsing.Chart.c_root_edges.ToString();
s_pack_rej = stats.Parsing.Unpacking.c_rejected_derivations.ToString();
}
String result_entry = String.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}",
num,
sent,
sent.Split(' ').Length,
s_c_parses,
s_sec,
s_c_unif,
s_forest,
s_pack_rej);
results.Add(result_entry);
//LexicalEntry.ReleaseAll(g.lex.lex_entries);
//foreach (GrammarRule r in g._grammar_rules)
// r.ReleaseExpanded();
if (mc.f_garbage)
Console.WriteLine(g.GarbageReport());
//g.loadtray.ResetMark();
//Console.WriteLine("{0:X8}", g.tm.last_mark_issued);
//Console.WriteLine(g.tm.GlbCacheInfo());
//foreach (var fi in g.tm.feat_arr.OrderByDescending(xx => xx.c_failures).Take(12))
// Console.Write("{0,9}", fi.feature.ToUpper());
//Console.WriteLine();
if (mc.f_profile && item_ix % 5 == 1)
{
Console.Out.WriteLineColorSync("$red pausing...");
Thread.Sleep(20000);
}
}
lock (g_results)
g_results.AddRange(results);
Interlocked.Decrement(ref c_tasks);
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Config config;
bool f_no_regress = false;
bool f_trees = false;
bool f_garbage = false;
bool f_tfs = false;
bool f_chart_dump = false;
bool f_quick_exit = false;
bool f_profile = false;
bool f_no_parse = false;
bool f_s100 = false;
bool f_no_report = false;
bool f_continuous = false;
bool f_check_dup_deriv = false;
bool f_item_info = false;
bool f_unif_regress = false;
int c_derivations_expected = int.MaxValue;
List<String> sentences_to_parse = new List<String>();
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void _Main(String[] args)
{
Console.OutputEncoding = Encoding.UTF8;
Console.ForegroundColor = (ConsoleColor)7;
config = new agree.Config();
for (int i = 0; i < args.Length; i++)
if (args[i].StartsWith("--"))
args[i] = args[i].Substring(1);
bool f_verbose = args.Contains("-v");
if (f_verbose)
args = args.Where(e => e != "-v").ToArray();
if (args.Contains("-help") || args.Contains("-?"))
{
Console.WriteLine(helpmsg);
return;
}
int arg_ix;
if ((arg_ix = Array.IndexOf<String>(args, "-singlethreaded")) != -1)
{
Console.WriteLine("disabling multi-threading");
config.system.MultiThreading = false;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-pipeline")) != -1 && arg_ix + 1 < args.Length)
{
int c_pipeline;
int.TryParse(args[arg_ix + 1], out c_pipeline);
if (c_pipeline == -1)
c_pipeline = int.MaxValue;
if (c_pipeline < 1)
c_pipeline = 1;
config.system.submitter.TaskCount = c_pipeline;
Console.WriteLine("pipeline: {0}", c_pipeline);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-timeout")) != -1 && ++arg_ix < args.Length)
{
double timeout;
if (!double.TryParse(args[arg_ix], out timeout))
timeout = double.NaN;
config.parser.ItemTimeoutSeconds = timeout;
Console.WriteLine("parser timeout: {0}", timeout);
args = args.Where((e, x) => x < arg_ix - 1 || x > arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-parseinfo")) != -1)
{
config.system.tw_debug = Console.Out;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-iteminfo")) != -1)
{
f_item_info = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-noregress")) != -1)
{
Console.WriteLine("disabling regression test");
f_no_regress = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-trees")) != -1)
{
f_trees = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-garbage")) != -1)
{
Grammar.f_garbage = true;
f_garbage = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-tfs")) != -1)
{
f_tfs = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-parsechart")) != -1)
{
f_chart_dump = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-quickexit")) != -1)
{
f_quick_exit = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-noreport")) != -1)
{
f_no_report = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-continuous")) != -1)
{
f_continuous = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-warmboot")) != -1)
return;
if ((arg_ix = Array.IndexOf<String>(args, "-profile")) != -1)
{
f_profile = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-quick-check")) != -1)
{
if (arg_ix + 1 < args.Length)
{
String su = args[arg_ix + 1].ToLower();
args = args.Where((e, x) => x != arg_ix + 1).ToArray();
if (su == "no" || su == "disable" || su == "off" || su == "none")
{
config.parser.f_qc_morph = false;
config.parser.f_qc_parse = false;
}
else if (su == "morph")
{
config.parser.f_qc_morph = true;
config.parser.f_qc_parse = false;
}
else if (su == "parse")
{
config.parser.f_qc_morph = false;
config.parser.f_qc_parse = true;
}
else if (su == "all")
{
config.parser.f_qc_morph = true;
config.parser.f_qc_parse = true;
}
}
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if (config.parser.f_qc_parse && config.parser.f_qc_morph)
Console.WriteLine("quick check: morph, parse");
else if (config.parser.f_qc_parse)
Console.WriteLine("quick check: parse");
else if (config.parser.f_qc_morph)
Console.WriteLine("quick check: morph");
else
Console.WriteLine("quick check: none");
if ((arg_ix = Array.IndexOf<String>(args, "-autotune")) != -1)
{
Console.WriteLine("enabling autotuning");
config.parser.f_autotune = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-noparse")) != -1)
{
f_no_parse = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-s100")) != -1)
{
f_s100 = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-check-duplicate-derivation")) != -1)
{
f_check_dup_deriv = true;
args = args.Where((e, x) => x != arg_ix).ToArray();
}
//if ((arg_ix = Array.IndexOf<String>(args, "-nounifregress")) != -1)
//{
// f_unif_regress = false;
// args = args.Where((e, x) => x != arg_ix).ToArray();
//}
arg_ix = 0;
while ((arg_ix = Array.IndexOf<String>(args, "-parse", arg_ix)) != -1 && arg_ix + 1 < args.Length)
{
sentences_to_parse.Add(args[arg_ix + 1]);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
arg_ix--;
}
if ((arg_ix = Array.IndexOf<String>(args, "-expected-parse-count")) != -1 && arg_ix + 1 < args.Length)
{
int.TryParse(args[arg_ix + 1], out c_derivations_expected);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-skipitems")) != -1 && arg_ix + 1 < args.Length)
{
int.TryParse(args[arg_ix + 1], out config.system.submitter.skipitems);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-takeitems")) != -1 && arg_ix + 1 < args.Length)
{
int.TryParse(args[arg_ix + 1], out config.system.submitter.takeitems);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-unifier")) != -1 && arg_ix + 1 < args.Length)
{
String su = args[arg_ix + 1].ToLower();
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
if (su == "incremental")
{
config.parser.unifier = Config.Parser.UnifierType.Incremental;
config.parser.checker = Config.Parser.UnifierType.Incremental;
}
else if (su == "qd")
{
config.parser.unifier = Config.Parser.UnifierType.qd;
config.parser.checker = Config.Parser.UnifierType.qd;
}
else if (su == "n-way")
{
config.parser.unifier = Config.Parser.UnifierType.n_way;
config.parser.checker = Config.Parser.UnifierType.n_way;
}
}
if (config.parser.unifier== Config.Parser.UnifierType.n_way)
config.parser.f_variadic_unpack = true;
Console.WriteLine("unifier: {0}", config.parser.unifier);
if ((arg_ix = Array.IndexOf<String>(args, "-checker")) != -1 && arg_ix + 1 < args.Length)
{
String su = args[arg_ix + 1].ToLower();
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
if (su == "incremental")
config.parser.checker = Config.Parser.UnifierType.Incremental;
else if (su == "qd")
config.parser.checker = Config.Parser.UnifierType.qd;
else if (su == "n-way")
config.parser.checker = Config.Parser.UnifierType.n_way;
}
Console.WriteLine("checker: {0}", config.parser.checker);
if ((arg_ix = Array.IndexOf<String>(args, "-variadic-unpack")) != -1)
{
int arg_ixix = arg_ix + 1;
if (arg_ixix >= args.Length)
{
Console.WriteLine("missing 'variadic-unpack' option");
Environment.Exit(0);
}
if (args[arg_ixix].ToLower() == "true")
config.parser.f_variadic_unpack = true;
else if (args[arg_ixix].ToLower() == "false")
config.parser.f_variadic_unpack = false;
else
{
Console.WriteLine("invalid 'variadic-unpack' option '{0}'", args[arg_ixix]);
Environment.Exit(0);
}
Console.WriteLine("variadic unpacking: {0}", config.parser.f_variadic_unpack);
args = args.Where((e, x) => x < arg_ix || x > arg_ixix).ToArray();
}
if (config.parser.f_variadic_unpack && config.parser.unifier != Config.Parser.UnifierType.n_way)
{
Console.Out.WriteLineColorSync("$red warning n-way unifier not selected with variadic unpacking");
}
if ((arg_ix = Array.IndexOf<String>(args, "-packing")) != -1)
{
int arg_ixix = arg_ix + 1;
if (arg_ixix >= args.Length)
{
Console.WriteLine("missing packing option");
Environment.Exit(0);
}
if (!Enum.TryParse(args[arg_ixix], true, out config.parser.packing))
{
Console.WriteLine("invalid packing option '{0}'", args[arg_ixix]);
Environment.Exit(0);
}
if (arg_ixix + 1 < args.Length && args[arg_ixix + 1].ToLower() == "only")
{
arg_ixix++;
config.parser.packing |= Config.Parser.PackingOpts.Only;
}
Console.WriteLine("packing: {0}", config.parser.packing);
args = args.Where((e, x) => x < arg_ix || x > arg_ixix).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-def-edge-hint")) != -1 && arg_ix + 1 < args.Length)
{
int.TryParse(args[arg_ix + 1], out Unification.def_edge_hint);
Console.WriteLine("default edge count hint: {0}", Unification.def_edge_hint);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-def-coref-hint")) != -1 && arg_ix + 1 < args.Length)
{
int.TryParse(args[arg_ix + 1], out Unification.def_edge_hint);
Console.WriteLine("default coref count hint: {0}", Unification.def_coref_hint);
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
if ((arg_ix = Array.IndexOf<String>(args, "-ctaskmax")) != -1 && arg_ix + 1 < args.Length)
{
int c_task_max;
if (int.TryParse(args[arg_ix + 1], out c_task_max))
{
config.parser.c_tasks_max = c_task_max;
Console.WriteLine("parser max tasks: {0}", c_task_max == -1 ? 1 : c_task_max);
}
args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray();
}
String filename = args.FirstOrDefault(e => e[0] != '-');
if (filename == null)
{
Console.WriteLine(helpmsg);
return;
}
String base_file = Path.GetFileNameWithoutExtension(filename).ToLower();
String ext = Path.GetExtension(filename).ToLower();
if (!f_no_regress)
Grammar.regress_file = base_file;
Grammar g;
using (TimingReport tr = new TimingReport(Console.Out, "Loading grammar file"))
g = new SysCommands.CmdTokLoadGrammar(SysObj.Instance, filename, base_file, config).Task.Result;
if (f_garbage)
Console.WriteLine(g.GarbageReport());
if (!f_no_parse)
{
for (int loop = 0; loop < 1 || f_continuous; loop++)
{
if (f_continuous)
Console.WriteLine("{0}", loop);
//if (g.loadtray is ConcurrentTray)
//{
// Console.Write("shrink tray");
// ((ConcurrentTray)g.loadtray).Shrink();
//}
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
if (base_file == "thai")
{
ParseThai(g);
}
else if (base_file == "erg")
{
new ParseErg(this).Parse(g);
}
else if (base_file == "g6morph")
{
ParseG6Morph(g);
}
}
}
Console.WriteLine();
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ParseControl DoParse(Grammar g, String sent)
{
var t = new SysCommands.CmdTokParse(g, sent).Task;
if (t == null)
return null;
return t.Result;
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary>
///
/// </summary>
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
static String helpmsg =
@"usage:
example:
agree [options] grammar_script_file
Options:
-noregress disable regression test
-timeout ss set parser timeout per sentence (sec.)
-trees show ASCII parse tree
-tfs show ASCII tfs
-garbage garbage edge report
-parseinfo verbose parsing output
-singlethreaded disable multi-threading
-skipitems nn skip over 'n' testsuite items
-takeitems nn process 'n' testsuite items
-ctaskmax nn maximum OS tasks allowed for parser
-parsechart dump HTML parse chart to 'chart_dump.html'
-parse ""Sentence to parse."" [-parse ""Another sentence"" ...]
";
}