#define TESTSUITE //#define TLDB using System; using System.IO; using System.Collections.Generic; using System.Linq; using System.Text; using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using miew.Concurrency; using miew.Tokenization; using miew.BitArray; using miew.String; using miew.String.Builder; using miew.Enumerable; using miew.Debugging; using miew.Math; using agree; //using tldb; #pragma warning disable 0618, 0414 class MainClass { /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// static void Main(string[] args) { Console.WriteLine("gcServer={0}, agree {1} sizeof(arr_tfs_entry)={2}", System.Runtime.GCSettings.IsServerGC, SysObj.build_type, System.Runtime.InteropServices.Marshal.SizeOf(typeof(arr_tfs_entry))); new MainClass()._Main(args); //Console.WriteLine("{0} {1} {2}", UnificationNWay.newway, UnificationNWay.oldway, (double)UnificationNWay.newway / (UnificationNWay.newway + UnificationNWay.oldway)); //Console.WriteLine("{0}", ParseControl.max_alloc); if (Debugger.IsAttached) { Console.WriteLine("done! press <enter> to exit"); Console.ReadLine(); } } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// public void ParseG6Morph(Grammar g) { int c_parses; using (TimingReport tr = new TimingReport(Console.Out, "Parsing...")) { c_parses = 0; try { using (ParseControl pc = DoParse(g, "give that cat to the dog")) { if (pc != null) { c_parses = pc.chart.CompletedEdges.Count(); Console.WriteLine(c_parses); } } } catch (AggregateException) { } } } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// public void ParseThai(Grammar g) { int c_parses; Tfs.MonospaceFormatter mf = null; if (f_tfs) mf = new Tfs.MonospaceFormatter(g.tm); using (TimingReport tr = new TimingReport(Console.Out, "Parsing...")) { //String s_sent = "เขา ไป ซื้อ ดอกไม้ ที่ ตลาด ไป เยี่ยม เพื่อน และ"; if (sentences_to_parse.Count > 0) { int i = 0; foreach (String l in sentences_to_parse) { c_parses = 0; try { TimingReport tr2 = null; if (f_trees || f_tfs || config.system.tw_debug != null) tr2 = new TimingReport(Console.Out, l); using (ParseControl pc = DoParse(g, l)) { if (pc != null) { c_parses = pc.chart.CompletedEdges.Count(); if (f_chart_dump) pc.chart.DumpChart(String.Format("chart_dump-{0}.html", i)); if (f_trees || f_tfs) { foreach (var px in pc.chart.AllDerivations) { if (f_trees) Console.WriteLine(g.nl.GetParseTree(pc.chart, px)); if (f_tfs) { Console.WriteLine(mf.FormatEdge(px.UnpackedTfs)); } } } } if (f_trees) tr2.Dispose(); } } catch (ParseException) { } catch (AggregateException) { } Console.WriteLine("{0} parses", c_parses); i++; } } else { foreach (String l in File.ReadAllLines("grammars/thai/testsuite.txt").Skip(config.system.submitter.skipitems).Take(config.system.submitter.takeitems)) { String[] rgs = l.Split('\t'); int id = int.Parse(rgs[0]); c_parses = 0; try { TimingReport tr2 = null; if (f_trees || f_tfs || f_item_info || config.system.tw_debug != null) tr2 = new TimingReport(Console.Out, String.Format("{0}", id)); using (ParseControl pc = DoParse(g, rgs[1])) { if (pc != null) { c_parses = pc.chart.CompletedEdges.Count(); if (f_chart_dump) pc.chart.DumpChart(String.Format("chart_dump-{0}.html", id)); if (f_trees || f_tfs) { foreach (var px in pc.chart.AllDerivations) { if (f_trees) Console.WriteLine(g.nl.GetParseTree(pc.chart, px)); if (f_tfs) { Console.WriteLine(mf.FormatEdge(px.UnpackedTfs)); } } } } } if (f_trees || f_tfs || f_item_info || config.system.tw_debug != null) tr2.Dispose(); } catch (ParseException) { } catch (AggregateException) { } int c_parses_exp = int.Parse(rgs[2]); if (c_parses != c_parses_exp) { String msg = String.Format("parser regression test failed for [{0}]\n{1} parses, expected {2}", rgs[1], c_parses, c_parses_exp); //throw new Exception(msg); Console.WriteLine(msg); } #if FCTC_STATS g.tm.TypePatternReport(); #endif //LexicalEntry.ReleaseAll(g.lex.lex_entries); //foreach (GrammarRule r in g._grammar_rules) // r.ReleaseExpanded(); if (f_garbage) Console.WriteLine(g.GarbageReport()); } } #if false Console.WriteLine("processing {0} test items", igts.Count); foreach (var testitem in igts) { Phrase phr = testitem.phr; //phr = db.GetPhrase(219755); String[] sent = phr.Parts.Select(e => e.FullDop.Thai(default(DbObj.LayoutOpts))).ToArray(); String s_sent = sent.StringJoin(" "); Console.WriteLine(); Console.WriteLine("Parsing: P{0} {1}", phr.Id, s_sent); try { using (ParseChart pc = DoParse(sent)) { c_parses = pc.CompletedParses.Count; } catch (AggregateException ex) { c_parses = 0; //Console.WriteLine(ex.Flatten().InnerException); } //Console.WriteLine(); //Console.WriteLine(g.GarbageReport()); Console.WriteLine("P{0} {1} Completed Parses: {2}", phr.Id, s_sent, c_parses); //Console.ReadLine(); sw_parsecounts.WriteLine("{0}\t{1}\t{2}", phr.Id, s_sent, c_parses); } #endif } //LexicalEntry.ReleaseAll(g.lex.lex_entries); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class ParseErg { int q = 0; MainClass mc; String[] original_lines; Tfs.MonospaceFormatter mf = null; String fdate; Grammar g; Dictionary<int, RegressInfo> hike_regress; ItemInfo[] rg_items; int gen_base; int c_unif_tot = 0; long ms_tot = 0; struct RegressInfo { public int id; public int readings; public int cunif; }; struct ItemInfo { public int id; public bool skip; public String text; }; public ParseErg(MainClass mc) { this.mc = mc; } public void Parse(Grammar g) { this.g = g; if (mc.f_tfs) mf = new Tfs.MonospaceFormatter(g.tm); fdate = String.Format("/programming/analytical-grammar/hike/{0:yyyyMMdd\\-hhmmss}-hike-results.txt", DateTime.Now); File.Delete(fdate); hike_regress = File.ReadAllLines("/programming/analytical-grammar/hike/agree-regress.txt").Select(oo => oo.Split('\t')).Select(vv => new RegressInfo { id = int.Parse(vv[0]), readings = vv[1] == "" ? 0 : int.Parse(vv[1]), cunif = vv[2] == "" ? 0 : int.Parse(vv[2]) }).ToDictionary(a => a.id); original_lines = System.IO.File.ReadAllLines(@"/programming/analytical-grammar/hike/hike-results-use.txt", Encoding.UTF8) .ToArray(); rg_items = original_lines .Select(vvv => { var wxyz = vvv.Split('\t'); var rsr = wxyz[0]; bool sk = rsr[0] == '#'; rsr = rsr.Trim('#'); return new ItemInfo { id = int.Parse(rsr), skip = sk, text = wxyz[1].Trim('\"'), /* sec = (Double)int.Parse(wxyz[2]) / 1000, readings = int.Parse(wxyz[3]) */ }; }).ToArray(); String[] rg_sent = rg_items.Skip(mc.config.system.submitter.skipitems).Take(mc.config.system.submitter.takeitems).Select(vsr => vsr.text).ToArray(); if (mc.sentences_to_parse.Count == 0) mc.sentences_to_parse.AddRange(rg_sent); if (mc.f_s100) { mc.sentences_to_parse.Clear(); mc.sentences_to_parse.AddRange(Enumerable.Repeat("Keep to the right in the track and yield one of the ski tracks if you meet someone.", 100)); } //if (!Debugger.IsAttached) //{ // using (var prelim = mc.DoParse(g, "This is a preliminary parse.")) // { // } //} //File.Delete("hike-full-regress.txt"); //if (mc.config.system.submitter.TaskCount > 1) // bar = new Barrier(0, b => // { // GC.Collect(); // GC.WaitForPendingFinalizers(); // GC.Collect(); // }); using (TimingReport tr = new TimingReport(Console.Out, "Parsing...")) { bw = Console.BufferWidth - 1; if (!mc.f_no_report) { Console.Out.WriteLineColorSync(hdr_a.PadLeft(bw)); Console.Out.WriteLineColorSync("$cyan " + hdr_b.PadLeft(bw)); } gen_base = GC.CollectionCount(2); if (mc.config.system.submitter.TaskCount == 1) ErgParseTask(); else { Task[] rgtasks = new Task[mc.config.system.submitter.TaskCount - 1]; for (int i = 0; i < mc.config.system.submitter.TaskCount - 1; i++) rgtasks[i] = Task.Factory.StartNew(ErgParseTask); ErgParseTask(); Task.WaitAll(rgtasks); } } File.WriteAllLines(fdate, g_results.OrderBy(ss => int.Parse(ss.Substring(0, ss.IndexOf('\t') + 1)))); } const string hdr_a = "+------totals------+ +------morph-------+ +--------------parse--------------+ +----pack----+ --unpack- --sys--"; const string hdr_b = " read wall unif %t unif % lx ana %t QC unif % pas-e act-e cmp-e eq fwd rev %t unif mem GC"; const int MIN_TRIM = 5; int bw; List<String> g_results = new List<string>(); int c_tasks = 0; //Barrier bar; public static string _PCT(double pct) { if (double.IsInfinity(pct)) return "oo"; if (double.IsNaN(pct)) return " "; if (pct < 1.0) return ((int)(pct * 100)).ToString(); return "%%"; } void ErgParseTask() { int tsk = Interlocked.Increment(ref c_tasks) - 1; List<String> results = new List<string>(); int item_ix; while ((item_ix = Interlocked.Increment(ref q) - 1) < mc.sentences_to_parse.Count) { String sent = mc.sentences_to_parse[item_ix]; //String disp_sent = sent.SubstringOrLess(0, 73); double mem = 0; int num; ParseControl.Stats stats = null; { int ix; if ((ix = Array.FindIndex(rg_items, oo => oo.text == sent)) != -1) { num = rg_items[ix].id; if (rg_items[ix].skip) goto skipit; item_ix = ix; } else { num = item_ix; } } StringBuilder report = new StringBuilder(); try { using (ParseControl ctrl = g.sub.Parse(sent).Result) { stats = ctrl.stats; if (mc.f_quick_exit) Environment.Exit(0); if (ctrl == null) { report.AppendFormat("$red null chart"); report.AppendLine(); } else { int c_deriv = ctrl.chart.AllDerivations.Count; #if DERIVATION_LIST foreach (var ez in ctrl.chart) { if (!ez.DerivedEdges.IsDistinct()) throw new Exception(); Console.WriteLine("{0}", ez); foreach (var qqqqs in ez.DerivedEdges) { if (qqqqs is PassiveEdge.CompletedParse) Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine(" {0}", qqqqs); Console.ResetColor(); } Console.WriteLine(); } #endif if (mc.f_chart_dump) ctrl.chart.DumpChart(String.Format("chart_dump-{0}.html", item_ix)); if (mc.f_trees || mc.f_tfs) { foreach (var px in ctrl.chart.AllDerivations) { if (mc.f_trees) Console.WriteLine(g.nl.GetParseTree(ctrl.chart, px)); if (mc.f_tfs) Console.WriteLine(mf.FormatEdge(px.UnpackedTfs)); } } #if false //foreach (var tr in pc.chart.CompletedParses.SelectMany(cp=>cp.Derivations.OfType<ParseChart.ParseTree>()).OrderBy(z => z.UnpackTask.Result.TfsHash())) foreach (var drv in pc.chart.CompletedParses.SelectMany(cp => cp.Derivations.OfType<ParseChart.ParseTree>()).OrderBy(z => z.DerivationHash)) { Console.WriteLine("#{0:X} =========================", drv.DerivationHash); Console.WriteLine(drv.TreeDisplay()); } Console.WriteLine(); #endif #if false System.IO.File.WriteAllText(String.Format("001_{0:X}.tfs", up2[9].id), up2[9].ToPathList(false)); System.IO.File.WriteAllText(String.Format("002_{0:X}.tfs", up2[10].id), up2[9].ToPathList(false)); #endif } } if (stats == null) goto skipit; //Console.Out.WriteLineColorSync("$red pausing..."); //Thread.Sleep(20000); String problems = ""; var a = hike_regress.FirstOrDefault(kvp => kvp.Key == num).Value; if (a.id == num) { int c_derivations = stats.Parsing.Unpacking.c_derivations; if (num != 0 && a.readings != c_derivations) { problems += String.Format("error: $red derivations {0}$ , expected {1}", c_derivations, a.readings); problems += Environment.NewLine; } else if (mc.c_derivations_expected != int.MaxValue && mc.c_derivations_expected != c_derivations) { problems += String.Format("error: $red derivations {0}$ , expected {1}", c_derivations, mc.c_derivations_expected); problems += Environment.NewLine; } if (mc.f_unif_regress && a.cunif != stats.Totals.Unification.c_attempted) problems += String.Format("error: unif {0}, expected {1}", stats.Totals.Unification.c_attempted, a.cunif); } if (!mc.f_no_report || problems != "") { //mem = (double)GC.GetTotalMemory(false) / (1024 * 1024 * 1024); mem = (double)Process.GetCurrentProcess().PrivateMemorySize64 / (1024 * 1024 * 1024); String gcinfo = String.Format("{0}", GC.CollectionCount(2) - gen_base); int trim_val = Math.Max(MIN_TRIM, bw - 9 - hdr_a.Length); String ds = sent.SubstringOrLess(0, trim_val); report.AppendFormat("{0,3:##0} {1,4:##0} $yellow {2}", item_ix, num, ds); report.Append(new String(' ', Math.Max(0, bw - 9 - hdr_a.Length - ds.Length))); // read wall unif. %t unif. ok lx ana %t unif. QC pas act cmp eq fwd rev %t unif. GB GC String rxt = String.Format("$magenta {0,5:G}$ {1,6:#.###} $green {2,7:G}$ $darkred {3,2}$ $green {4,5:G}$ {5,2} {6,3:G} {7,4:G} $darkred {8,2}$ {9,2}$green {10,7:G}$ {11,2} {12,6:G} {13,6:G} {14,5:G}{15,5:G}{16,5:G}{17,5:G} $darkred {18,2}$ $green {19,6:G}$ {20:#.##} {21,2}", stats.Parsing.Unpacking.c_derivations, stats.Totals.sec_time, stats.Totals.Unification.c_attempted, _PCT(stats.Morphology.TimeRatio), stats.Morphology.Unification.c_attempted, _PCT(stats.Morphology.Unification.SuccessRatio), stats.Morphology.c_lexical_transforms, stats.Morphology.c_analysis_stacks, _PCT(stats.Parsing.Chart.TimeRatio), _PCT(stats.Parsing.QuickCheck.PercentAvoided), stats.Parsing.Unification.c_attempted, _PCT(stats.Parsing.Unification.SuccessRatio), stats.Parsing.Chart.c_passive_edges, stats.Parsing.Chart.c_active_edges, stats.Parsing.Chart.c_root_edges, stats.Parsing.Packing.c_equivalence, stats.Parsing.Packing.c_proactive, stats.Parsing.Packing.c_retroactive, _PCT(stats.Parsing.Unpacking.TimeRatio), stats.Parsing.Unpacking.Unification.c_attempted, mem.ToString("#.##").TrimEndOrPadLeft(4), gcinfo); report.AppendLine(rxt); report.AppendLine(problems); } //if (PassiveEdge.Derived.qc > 0) //{ // Console.Write("{0} ", PassiveEdge.Derived.qc); // PassiveEdge.Derived.qc = 0; //} //if (ctrl.chart.hs.Count > 0) // report.AppendLine(" " + ctrl.chart.hs.StringJoin(" ")); //String fr = String.Format("{0}\t{1}\t{2}", num, c_parses, up2 // .Select(tz => tz.TfsHash()) // .OrderBy(tz => tz) // .Select(tz => tz.ToString("X")) // .StringJoin("\t")); //File.AppendAllText("hike-full-regress.txt", fr + Environment.NewLine); } //Console.WriteLine(g.tm.GlbCacheInfo()); catch (Exception ex) { report.AppendFormat("{0,3:##0} {1,3:##0} $yellow {2} ", item_ix, num, sent); if (ex is AggregateException) ex = ((AggregateException)ex).Flatten().InnerExceptions.FirstOrDefault(x => !(x is AggregateException)) ?? ex; report.AppendFormatLine("\r\n$red {0} {1}$ \r\n", ex.GetType().Name, ex.Message); if (!(ex is ParseException)) { report.AppendLine(ex.StackTrace); lock (Console.Out) { Console.Out.WriteLineColorSync(report.ToString()); Console.ResetColor(); Console.Out.Flush(); } throw ex; } } #if FCTC_STATS g.tm.TypePatternReport(); #endif if (report.Length > 0) { if (mc.config.system.submitter.TaskCount > 1 && Monitor.TryEnter(Console.Out)) { foreach (string lll in report.ToString().Split(new Char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) Console.Out.WriteLineColorSync(lll); Monitor.Exit(Console.Out); } else Task.Factory.StartNew(() => { Monitor.Enter(Console.Out); foreach (string lll in report.ToString().Split(new Char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) Console.Out.WriteLineColorSync(lll); Monitor.Exit(Console.Out); }); } //if (bar != null) //{ // if (bar.ParticipantCount > 0) // bar.SignalAndWait(); // else if (mem > 16.0) // { // bar.AddParticipants(c_tasks); // bar.SignalAndWait(); // bar.RemoveParticipants(c_tasks); // } //} skipit: String s_c_parses; String s_sec; String s_c_unif; String s_forest; String s_pack_rej; if (stats == null) { s_pack_rej = s_forest = s_sec = s_c_unif = s_c_parses = "".Quotes(); } else { s_c_unif = stats.Totals.Unification.c_attempted.ToString(); s_sec = stats.Totals.sec_time.ToString(); s_c_parses = stats.Parsing.Unpacking.c_derivations.ToString(); s_forest = stats.Parsing.Chart.c_root_edges.ToString(); s_pack_rej = stats.Parsing.Unpacking.c_rejected_derivations.ToString(); } String result_entry = String.Format("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}", num, sent, sent.Split(' ').Length, s_c_parses, s_sec, s_c_unif, s_forest, s_pack_rej); results.Add(result_entry); //LexicalEntry.ReleaseAll(g.lex.lex_entries); //foreach (GrammarRule r in g._grammar_rules) // r.ReleaseExpanded(); if (mc.f_garbage) Console.WriteLine(g.GarbageReport()); //g.loadtray.ResetMark(); //Console.WriteLine("{0:X8}", g.tm.last_mark_issued); //Console.WriteLine(g.tm.GlbCacheInfo()); //foreach (var fi in g.tm.feat_arr.OrderByDescending(xx => xx.c_failures).Take(12)) // Console.Write("{0,9}", fi.feature.ToUpper()); //Console.WriteLine(); if (mc.f_profile && item_ix % 5 == 1) { Console.Out.WriteLineColorSync("$red pausing..."); Thread.Sleep(20000); } } lock (g_results) g_results.AddRange(results); Interlocked.Decrement(ref c_tasks); } } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// Config config; bool f_no_regress = false; bool f_trees = false; bool f_garbage = false; bool f_tfs = false; bool f_chart_dump = false; bool f_quick_exit = false; bool f_profile = false; bool f_no_parse = false; bool f_s100 = false; bool f_no_report = false; bool f_continuous = false; bool f_check_dup_deriv = false; bool f_item_info = false; bool f_unif_regress = false; int c_derivations_expected = int.MaxValue; List<String> sentences_to_parse = new List<String>(); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// void _Main(String[] args) { Console.OutputEncoding = Encoding.UTF8; Console.ForegroundColor = (ConsoleColor)7; config = new agree.Config(); for (int i = 0; i < args.Length; i++) if (args[i].StartsWith("--")) args[i] = args[i].Substring(1); bool f_verbose = args.Contains("-v"); if (f_verbose) args = args.Where(e => e != "-v").ToArray(); if (args.Contains("-help") || args.Contains("-?")) { Console.WriteLine(helpmsg); return; } int arg_ix; if ((arg_ix = Array.IndexOf<String>(args, "-singlethreaded")) != -1) { Console.WriteLine("disabling multi-threading"); config.system.MultiThreading = false; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-pipeline")) != -1 && arg_ix + 1 < args.Length) { int c_pipeline; int.TryParse(args[arg_ix + 1], out c_pipeline); if (c_pipeline == -1) c_pipeline = int.MaxValue; if (c_pipeline < 1) c_pipeline = 1; config.system.submitter.TaskCount = c_pipeline; Console.WriteLine("pipeline: {0}", c_pipeline); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-timeout")) != -1 && ++arg_ix < args.Length) { double timeout; if (!double.TryParse(args[arg_ix], out timeout)) timeout = double.NaN; config.parser.ItemTimeoutSeconds = timeout; Console.WriteLine("parser timeout: {0}", timeout); args = args.Where((e, x) => x < arg_ix - 1 || x > arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-parseinfo")) != -1) { config.system.tw_debug = Console.Out; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-iteminfo")) != -1) { f_item_info = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-noregress")) != -1) { Console.WriteLine("disabling regression test"); f_no_regress = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-trees")) != -1) { f_trees = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-garbage")) != -1) { Grammar.f_garbage = true; f_garbage = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-tfs")) != -1) { f_tfs = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-parsechart")) != -1) { f_chart_dump = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-quickexit")) != -1) { f_quick_exit = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-noreport")) != -1) { f_no_report = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-continuous")) != -1) { f_continuous = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-warmboot")) != -1) return; if ((arg_ix = Array.IndexOf<String>(args, "-profile")) != -1) { f_profile = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-quick-check")) != -1) { if (arg_ix + 1 < args.Length) { String su = args[arg_ix + 1].ToLower(); args = args.Where((e, x) => x != arg_ix + 1).ToArray(); if (su == "no" || su == "disable" || su == "off" || su == "none") { config.parser.f_qc_morph = false; config.parser.f_qc_parse = false; } else if (su == "morph") { config.parser.f_qc_morph = true; config.parser.f_qc_parse = false; } else if (su == "parse") { config.parser.f_qc_morph = false; config.parser.f_qc_parse = true; } else if (su == "all") { config.parser.f_qc_morph = true; config.parser.f_qc_parse = true; } } args = args.Where((e, x) => x != arg_ix).ToArray(); } if (config.parser.f_qc_parse && config.parser.f_qc_morph) Console.WriteLine("quick check: morph, parse"); else if (config.parser.f_qc_parse) Console.WriteLine("quick check: parse"); else if (config.parser.f_qc_morph) Console.WriteLine("quick check: morph"); else Console.WriteLine("quick check: none"); if ((arg_ix = Array.IndexOf<String>(args, "-autotune")) != -1) { Console.WriteLine("enabling autotuning"); config.parser.f_autotune = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-noparse")) != -1) { f_no_parse = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-s100")) != -1) { f_s100 = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-check-duplicate-derivation")) != -1) { f_check_dup_deriv = true; args = args.Where((e, x) => x != arg_ix).ToArray(); } //if ((arg_ix = Array.IndexOf<String>(args, "-nounifregress")) != -1) //{ // f_unif_regress = false; // args = args.Where((e, x) => x != arg_ix).ToArray(); //} arg_ix = 0; while ((arg_ix = Array.IndexOf<String>(args, "-parse", arg_ix)) != -1 && arg_ix + 1 < args.Length) { sentences_to_parse.Add(args[arg_ix + 1]); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); arg_ix--; } if ((arg_ix = Array.IndexOf<String>(args, "-expected-parse-count")) != -1 && arg_ix + 1 < args.Length) { int.TryParse(args[arg_ix + 1], out c_derivations_expected); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-skipitems")) != -1 && arg_ix + 1 < args.Length) { int.TryParse(args[arg_ix + 1], out config.system.submitter.skipitems); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-takeitems")) != -1 && arg_ix + 1 < args.Length) { int.TryParse(args[arg_ix + 1], out config.system.submitter.takeitems); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-unifier")) != -1 && arg_ix + 1 < args.Length) { String su = args[arg_ix + 1].ToLower(); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); if (su == "incremental") { config.parser.unifier = Config.Parser.UnifierType.Incremental; config.parser.checker = Config.Parser.UnifierType.Incremental; } else if (su == "qd") { config.parser.unifier = Config.Parser.UnifierType.qd; config.parser.checker = Config.Parser.UnifierType.qd; } else if (su == "n-way") { config.parser.unifier = Config.Parser.UnifierType.n_way; config.parser.checker = Config.Parser.UnifierType.n_way; } } if (config.parser.unifier== Config.Parser.UnifierType.n_way) config.parser.f_variadic_unpack = true; Console.WriteLine("unifier: {0}", config.parser.unifier); if ((arg_ix = Array.IndexOf<String>(args, "-checker")) != -1 && arg_ix + 1 < args.Length) { String su = args[arg_ix + 1].ToLower(); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); if (su == "incremental") config.parser.checker = Config.Parser.UnifierType.Incremental; else if (su == "qd") config.parser.checker = Config.Parser.UnifierType.qd; else if (su == "n-way") config.parser.checker = Config.Parser.UnifierType.n_way; } Console.WriteLine("checker: {0}", config.parser.checker); if ((arg_ix = Array.IndexOf<String>(args, "-variadic-unpack")) != -1) { int arg_ixix = arg_ix + 1; if (arg_ixix >= args.Length) { Console.WriteLine("missing 'variadic-unpack' option"); Environment.Exit(0); } if (args[arg_ixix].ToLower() == "true") config.parser.f_variadic_unpack = true; else if (args[arg_ixix].ToLower() == "false") config.parser.f_variadic_unpack = false; else { Console.WriteLine("invalid 'variadic-unpack' option '{0}'", args[arg_ixix]); Environment.Exit(0); } Console.WriteLine("variadic unpacking: {0}", config.parser.f_variadic_unpack); args = args.Where((e, x) => x < arg_ix || x > arg_ixix).ToArray(); } if (config.parser.f_variadic_unpack && config.parser.unifier != Config.Parser.UnifierType.n_way) { Console.Out.WriteLineColorSync("$red warning n-way unifier not selected with variadic unpacking"); } if ((arg_ix = Array.IndexOf<String>(args, "-packing")) != -1) { int arg_ixix = arg_ix + 1; if (arg_ixix >= args.Length) { Console.WriteLine("missing packing option"); Environment.Exit(0); } if (!Enum.TryParse(args[arg_ixix], true, out config.parser.packing)) { Console.WriteLine("invalid packing option '{0}'", args[arg_ixix]); Environment.Exit(0); } if (arg_ixix + 1 < args.Length && args[arg_ixix + 1].ToLower() == "only") { arg_ixix++; config.parser.packing |= Config.Parser.PackingOpts.Only; } Console.WriteLine("packing: {0}", config.parser.packing); args = args.Where((e, x) => x < arg_ix || x > arg_ixix).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-def-edge-hint")) != -1 && arg_ix + 1 < args.Length) { int.TryParse(args[arg_ix + 1], out Unification.def_edge_hint); Console.WriteLine("default edge count hint: {0}", Unification.def_edge_hint); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-def-coref-hint")) != -1 && arg_ix + 1 < args.Length) { int.TryParse(args[arg_ix + 1], out Unification.def_edge_hint); Console.WriteLine("default coref count hint: {0}", Unification.def_coref_hint); args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } if ((arg_ix = Array.IndexOf<String>(args, "-ctaskmax")) != -1 && arg_ix + 1 < args.Length) { int c_task_max; if (int.TryParse(args[arg_ix + 1], out c_task_max)) { config.parser.c_tasks_max = c_task_max; Console.WriteLine("parser max tasks: {0}", c_task_max == -1 ? 1 : c_task_max); } args = args.Where((e, x) => x < arg_ix || x > arg_ix + 1).ToArray(); } String filename = args.FirstOrDefault(e => e[0] != '-'); if (filename == null) { Console.WriteLine(helpmsg); return; } String base_file = Path.GetFileNameWithoutExtension(filename).ToLower(); String ext = Path.GetExtension(filename).ToLower(); if (!f_no_regress) Grammar.regress_file = base_file; Grammar g; using (TimingReport tr = new TimingReport(Console.Out, "Loading grammar file")) g = new SysCommands.CmdTokLoadGrammar(SysObj.Instance, filename, base_file, config).Task.Result; if (f_garbage) Console.WriteLine(g.GarbageReport()); if (!f_no_parse) { for (int loop = 0; loop < 1 || f_continuous; loop++) { if (f_continuous) Console.WriteLine("{0}", loop); //if (g.loadtray is ConcurrentTray) //{ // Console.Write("shrink tray"); // ((ConcurrentTray)g.loadtray).Shrink(); //} GC.Collect(); GC.WaitForPendingFinalizers(); GC.Collect(); if (base_file == "thai") { ParseThai(g); } else if (base_file == "erg") { new ParseErg(this).Parse(g); } else if (base_file == "g6morph") { ParseG6Morph(g); } } } Console.WriteLine(); } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ParseControl DoParse(Grammar g, String sent) { var t = new SysCommands.CmdTokParse(g, sent).Task; if (t == null) return null; return t.Result; } /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /// <summary> /// /// </summary> /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// static String helpmsg = @"usage: example: agree [options] grammar_script_file Options: -noregress disable regression test -timeout ss set parser timeout per sentence (sec.) -trees show ASCII parse tree -tfs show ASCII tfs -garbage garbage edge report -parseinfo verbose parsing output -singlethreaded disable multi-threading -skipitems nn skip over 'n' testsuite items -takeitems nn process 'n' testsuite items -ctaskmax nn maximum OS tasks allowed for parser -parsechart dump HTML parse chart to 'chart_dump.html' -parse ""Sentence to parse."" [-parse ""Another sentence"" ...] "; }