From 361d2e2f7448038355b337ee4f7191a28c20755a Mon Sep 17 00:00:00 2001 From: David Chen Date: Sat, 12 Sep 2015 01:32:26 +0000 Subject: Add Jsonnet rules to Bazel. RELNOTES: Add Jsonnet rules to Bazel -- MOS_MIGRATED_REVID=102895524 --- examples/jsonnet/BUILD | 41 ++++++++++++++++++++++ examples/jsonnet/intersection.jsonnet | 60 ++++++++++++++++++++++++++++++++ examples/jsonnet/shell-workflows.jsonnet | 21 +++++++++++ examples/jsonnet/wordcount.jsonnet | 59 +++++++++++++++++++++++++++++++ examples/jsonnet/workflow.jsonnet | 46 ++++++++++++++++++++++++ 5 files changed, 227 insertions(+) create mode 100644 examples/jsonnet/BUILD create mode 100644 examples/jsonnet/intersection.jsonnet create mode 100644 examples/jsonnet/shell-workflows.jsonnet create mode 100644 examples/jsonnet/wordcount.jsonnet create mode 100644 examples/jsonnet/workflow.jsonnet (limited to 'examples') diff --git a/examples/jsonnet/BUILD b/examples/jsonnet/BUILD new file mode 100644 index 0000000000..3c829df876 --- /dev/null +++ b/examples/jsonnet/BUILD @@ -0,0 +1,41 @@ +package(default_visibility = ["//visibility:public"]) + +load("/tools/build_defs/jsonnet/jsonnet", "jsonnet_library", "jsonnet_to_json") + +jsonnet_library( + name = "workflow", + srcs = ["workflow.jsonnet"], +) + +jsonnet_to_json( + name = "wordcount", + src = "wordcount.jsonnet", + outs = ["wordcount.json"], + deps = [":workflow"], +) + +jsonnet_to_json( + name = "intersection", + src = "intersection.jsonnet", + outs = ["intersection.json"], + deps = [":workflow"], +) + +jsonnet_library( + name = "shell-workflows-lib", + srcs = [ + "intersection.jsonnet", + "wordcount.jsonnet", + ], + deps = [":workflow"], +) + +jsonnet_to_json( + name = "shell-workflows", + src = "shell-workflows.jsonnet", + outs = [ + "intersection-workflow.json", + "wordcount-workflow.json", + ], + deps = [":shell-workflows-lib"], +) diff --git a/examples/jsonnet/intersection.jsonnet b/examples/jsonnet/intersection.jsonnet new file mode 100644 index 0000000000..e0f2fd1224 --- /dev/null +++ b/examples/jsonnet/intersection.jsonnet @@ -0,0 +1,60 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +local workflow = import "examples/jsonnet/workflow.jsonnet"; + +// Workflow that performs an intersection of two files using shell commands. +{ + intersection: workflow.Workflow { + jobs: { + local input_file1 = "/tmp/list1", + local input_file2 = "/tmp/list2", + local sorted_file1 = "/tmp/list1_sorted", + local sorted_file2 = "/tmp/list2_sorted", + local intersection = "/tmp/intersection", + + SortJob:: workflow.ShJob { + input_file:: "", + output_file:: "", + command: "sort %s > %s" % [self.input_file, self.output_file], + inputs: [self.input_file], + outputs: [self.output_file], + }, + + sort_file1: self.SortJob { + input_file:: input_file1, + output_file:: sorted_file1, + }, + + sort_file2: self.SortJob { + input_file:: input_file2, + output_file:: sorted_file2, + }, + + intersect: workflow.ShJob { + deps: [ + ":sort_file1", + ":sort_file2", + ], + command: "comm -12 %s %s > %s" % + [sorted_file1, sorted_file2, intersection], + inputs: [ + sorted_file1, + sorted_file2, + ], + outputs: [intersection], + }, + } + } +} diff --git a/examples/jsonnet/shell-workflows.jsonnet b/examples/jsonnet/shell-workflows.jsonnet new file mode 100644 index 0000000000..d5e36a07ec --- /dev/null +++ b/examples/jsonnet/shell-workflows.jsonnet @@ -0,0 +1,21 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +local wordcount = import "examples/jsonnet/wordcount.jsonnet"; +local intersection = import "examples/jsonnet/intersection.jsonnet"; + +{ + "wordcount-workflow.json": wordcount, + "intersection-workflow.json": intersection, +} diff --git a/examples/jsonnet/wordcount.jsonnet b/examples/jsonnet/wordcount.jsonnet new file mode 100644 index 0000000000..e816d446b4 --- /dev/null +++ b/examples/jsonnet/wordcount.jsonnet @@ -0,0 +1,59 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +local workflow = import "examples/jsonnet/workflow.jsonnet"; + +// Workflow that performs a wordcount using shell commands. +{ + wordcount: workflow.Workflow { + retries: 12, + schedule: workflow.Schedule { + start_date: "2015-11-15", + start_time: "17:30", + repeat_frequency: 1, + repeat_type: "week", + }, + jobs: { + local input_file = "/tmp/passage_test", + local tokens_file = "/tmp/tokens", + local sorted_tokens_file = "/tmp/sorted_tokens", + local counts_file = "/tmp/counts", + + // Reads the input file and produces an output file with one word per + // line. + tokenize: workflow.ShJob { + command: "tr ' ' '\n' < %s > %s" % [input_file, tokens_file], + inputs: [input_file], + outputs: [tokens_file], + }, + + // Takes the tokens file and produces a file with the tokens sorted. + sort: workflow.ShJob { + deps: [":tokenize"], + command: "sort %s > %s" % [tokens_file, sorted_tokens_file], + inputs: [tokens_file], + outputs: [sorted_tokens_file], + }, + + // Takes the file containing sorted tokens and produces a file containing + // the counts for each word. + count: workflow.ShJob { + deps: [":sort"], + command: "uniq -c %s > %s" % [sorted_tokens_file, counts_file], + inputs: [sorted_tokens_file], + outputs: [counts_file], + }, + } + } +} diff --git a/examples/jsonnet/workflow.jsonnet b/examples/jsonnet/workflow.jsonnet new file mode 100644 index 0000000000..bca2bd7e16 --- /dev/null +++ b/examples/jsonnet/workflow.jsonnet @@ -0,0 +1,46 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Configuration for a hypothetical workflow scheduling system. +{ + // Configuration for a workflow. + Workflow:: { + schedule: {}, + retries: 5, + jobs: {}, + }, + + // Scheduling configuration for a workflow. + Schedule:: { + start_date: "", + start_time: "", + repeat_frequency: 0, + repeat_type: "", + }, + + // Base configuration for a Job in a workflow. + Job:: { + type: "base", + deps: [], + inputs: [], + outputs: [], + }, + + // Configuration for a job that runs a shell command. + ShJob:: self.Job { + type: "sh", + command: "", + vars: {}, + } +} -- cgit v1.2.3