aboutsummaryrefslogtreecommitdiffhomepage
path: root/tools/lua/ngrams_aggregate.lua
diff options
context:
space:
mode:
authorGravatar borenet <borenet@google.com>2015-06-25 07:47:40 -0700
committerGravatar Commit bot <commit-bot@chromium.org>2015-06-25 07:47:41 -0700
commit93558dc74be4a92a8dfba87fcffd4441a37c0fc8 (patch)
treee5f3136fc4b8cde416059146061794247efecf1f /tools/lua/ngrams_aggregate.lua
parentfb8d6884e0e01d0c2f8596adf5af1efb0d08de7e (diff)
Add lua scripts for generating n-grams from SKPs
Diffstat (limited to 'tools/lua/ngrams_aggregate.lua')
-rw-r--r--tools/lua/ngrams_aggregate.lua22
1 files changed, 22 insertions, 0 deletions
diff --git a/tools/lua/ngrams_aggregate.lua b/tools/lua/ngrams_aggregate.lua
new file mode 100644
index 0000000000..f8c9338149
--- /dev/null
+++ b/tools/lua/ngrams_aggregate.lua
@@ -0,0 +1,22 @@
+-- Aggregate the output from ngrams.lua.
+
+-- Get the data from all shards.
+counts = {}
+dofile("/tmp/lua-output")
+
+-- Put the data into a sortable "array".
+countArray = {}
+for ngram, count in pairs(counts) do
+ table.insert(countArray, {count, ngram})
+end
+
+-- Sort the data.
+function compare(a, b)
+ return a[1] > b[1]
+end
+table.sort(countArray, compare)
+
+-- Write the result.
+for i, countPair in ipairs(countArray) do
+ io.write(countPair[1], "\t", countPair[2], "\n")
+end