diff options
157 files changed, 6116 insertions, 13416 deletions
diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 8a0fcea688..acbc701056 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -134,7 +134,7 @@ function bazel_build() { cp bazel-genfiles/bazel-distfile.zip $1/bazel-${release_label}-dist.zip fi fi - cp bazel-genfiles/site/jekyll-tree.tar $1/www.bazel.build.tar + cp bazel-genfiles/site/jekyll-tree.tar $1/docs.bazel.build.tar cp bazel-bin/src/tools/benchmark/webapp/site.tar $1/perf.bazel.build.tar.nobuild cp bazel-genfiles/scripts/packages/README.md $1/README.md fi diff --git a/scripts/serve-docs.sh b/scripts/serve-docs.sh index edbe41384d..8d06d65e96 100755 --- a/scripts/serve-docs.sh +++ b/scripts/serve-docs.sh @@ -59,7 +59,7 @@ build_static() { build_and_serve() { build_tree - echo "Serving bazel.build site at $HOST:$PORT" + echo "Serving docs.bazel.build site at $HOST:$PORT" jekyll serve --host "$HOST" --detach --quiet --port "$PORT" --source "$WORKING_DIR" } diff --git a/site/BUILD b/site/BUILD index acada35d77..1672988fdb 100644 --- a/site/BUILD +++ b/site/BUILD @@ -2,15 +2,15 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_tar") exports_files( [ - "versions/master/docs/bazel-user-manual.html", + "docs/bazel-user-manual.html", "command-line-reference-prefix.html", "command-line-reference-suffix.html", ], ) DOT_GRAPH_HTML_FILES = [ - "versions/master/docs/query.html", - "versions/master/docs/build-ref.html", + "docs/query.html", + "docs/build-ref.html", ] filegroup( diff --git a/site/_includes/blog-sidebar.html b/site/_includes/blog-sidebar.html deleted file mode 100644 index f56362318a..0000000000 --- a/site/_includes/blog-sidebar.html +++ /dev/null @@ -1,20 +0,0 @@ - <div class="col-lg-3"> - <h2>Twitter</h2> - <p>For more frequent updates, follow us on Twitter.</p> - <p><a class="twitter-follow-button" href="https://twitter.com/bazelbuild" data-show-count="false" data-size="large">Follow @bazelbuild</a></p> - <h2>Discuss</h2> - <p>Join the discussion at our <a href="https://groups.google.com/forum/#!forum/bazel-discuss">mailing list</a>.</p> - <h2>Subscribe</h2> - <p>Subscribe to our blog via the <a href="/blog/feed.xml">RSS Feed</a> or via email:</p> - <div class="well"> - <form action="https://feedburner.google.com/fb/a/mailverify" method="post" target="popupwindow" onsubmit="window.open('https://feedburner.google.com/fb/a/mailverify?uri=BazelBlog', 'popupwindow', 'scrollbars=yes,width=550,height=520');return true"> - <div class="form-group"> - <input type="text" class="form-control" name="email" placeholder="name@email.com"> - </div> - <input type="hidden" value="BazelBlog" name="uri"> - <input type="hidden" name="loc" value="en_US"> - <button type="submit" class="btn btn-primary btn-block">Subscribe</button> - </form> - <p>Delivered by <a href="https://feedburner.google.com" target="_blank">FeedBurner</a></p> - </div> - </div> diff --git a/site/_includes/blog-twitter-js.html b/site/_includes/blog-twitter-js.html deleted file mode 100644 index c6dfa460da..0000000000 --- a/site/_includes/blog-twitter-js.html +++ /dev/null @@ -1,6 +0,0 @@ -<!-- -JavaScript used by the Follow @bazelbuild on Twitter button on the blog sidebar. -Code from: https://dev.twitter.com/web/follow-button ---> - -<script>window.twttr=(function(d,s,id){var js,fjs=d.getElementsByTagName(s)[0],t=window.twttr||{};if(d.getElementById(id))return;js=d.createElement(s);js.id=id;js.src="https://platform.twitter.com/widgets.js";fjs.parentNode.insertBefore(js,fjs);t._e=[];t.ready=function(f){t._e.push(f);};return t;}(document,"script","twitter-wjs"));</script> diff --git a/site/_layouts/blog.html b/site/_layouts/blog.html deleted file mode 100644 index 685c8a51c2..0000000000 --- a/site/_layouts/blog.html +++ /dev/null @@ -1,30 +0,0 @@ ---- -nav: blog ---- - -<!DOCTYPE html> -<html lang="en" itemscope itemtype="https://schema.org/WebPage"> - {% include head.html %} - <body> - {% include header.html %} - - <div class="page-title-bar"> - <div class="container"> - <h1>Bazel Blog</h1> - </div> - </div> - - <div class="container vpad"> - <div class="row"> - <div class="col-lg-9"> - {{ content }} - </div> - {% include blog-sidebar.html %} - </div> - </div> - - {% include footer.html %} - {% include blog-twitter-js.html %} - </body> -</html> - diff --git a/site/_layouts/contribute.html b/site/_layouts/contribute.html deleted file mode 100644 index c9657cfa06..0000000000 --- a/site/_layouts/contribute.html +++ /dev/null @@ -1,55 +0,0 @@ ---- -nav: contribute ---- - -<!DOCTYPE html> -<html lang="en" itemscope itemtype="https://schema.org/WebPage"> - {% include head.html %} - <body> - {% include header.html %} - - <div class="page-title-bar"> - <div class="container"> - <h1>Contribute!</h1> - </div> - </div> - - <div class="container vpad"> - <div class="row"> - <div class="col-lg-3"> - <a class="btn btn-default btn-lg btn-block sidebar-toggle" - data-toggle="collapse" href="#sidebar-nav" aria-expanded="false" - aria-controls="sidebar-nav"> - <i class="glyphicon glyphicon-menu-hamburger"></i> Navigation - </a> - <nav class="sidebar collapse" id="sidebar-nav"> - <ul class="sidebar-nav"> - <li><a href="/contributing.html">Contributing to Bazel</a></li> - <li><a href="https://github.com/bazelbuild/bazel/wiki/Bazel-Users">Who's Using Bazel</a></li> - <li><a href="/roadmap.html">Roadmap</a></li> - <li><a href="/governance.html">Governance</a></li> - </ul> - <h3>Technical Docs</h3> - <ul class="sidebar-nav"> - <li><a href="/designs/skyframe.html">Skyframe</a></li> - <li><a href="/designs/index.html">Design Documents</a></li> - <li><a href="/windows-chocolatey-maintenance.html">Maintaining Bazel Chocolatey package on Windows</a></li> - </ul> - <h3>External Resources</h3> - <ul class="sidebar-nav"> - <li><a href="https://github.com/bazelbuild/bazel">GitHub</a></li> - <li><a href="https://groups.google.com/forum/#!forum/bazel-dev">Developer mailing list</a></li> - <li><a href="https://bazel-review.googlesource.com">Gerrit</a></li> - <li><a href="irc://irc.freenode.net/bazel">IRC</a><li> - </ul> - </nav> - </div> - <div class="col-lg-9"> - {{ content }} - </div> - </div> - </div> - - {% include footer.html %} - </body> -</html> diff --git a/site/_layouts/home.html b/site/_layouts/home.html deleted file mode 100644 index a68db67005..0000000000 --- a/site/_layouts/home.html +++ /dev/null @@ -1,11 +0,0 @@ -<!DOCTYPE html> -<html lang="en" itemscope itemtype="https://schema.org/WebPage"> - {% include head.html %} - <body class="body-home"> - {% include header.html %} - - {{ content }} - - {% include footer.html %} - </body> -</html> diff --git a/site/_layouts/posts.html b/site/_layouts/posts.html deleted file mode 100644 index b02dedee41..0000000000 --- a/site/_layouts/posts.html +++ /dev/null @@ -1,54 +0,0 @@ ---- -nav: blog -enable_comments: true ---- - - -<!DOCTYPE html> -<html lang="en" itemscope itemtype="https://schema.org/WebPage"> - {% include head.html %} - <body> - {% include header.html %} - - <div class="page-title-bar"> - <div class="container"> - <h1>Bazel Blog</h1> - </div> - </div> - - <div class="container vpad"> - <div class="row"> - <div class="col-lg-9"> - <div class="blog-post"> - <h1 class="blog-post-title">{{ page.title }}</h1> - <div class="blog-post-meta"> - <span class="text-muted">{{ page.date | date_to_long_string }}</a> - </div> - <div class="blog-post-content"> - {{ content }} - </div> - {% if page.enable_comments %} - <div class="blog-post-comments"> - <div id="disqus_thread"></div> - <script type="text/javascript"> - var disqus_shortname = 'googlebazel'; - - (function() { - var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true; - dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js'; - (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq); - })(); - </script> - <noscript>Please enable JavaScript to view the <a href="https://disqus.com/?ref_noscript" rel="nofollow">comments powered by Disqus.</a></noscript> - </div> - {% endif %} - </div> - </div> - {% include blog-sidebar.html %} - </div> - </div> - - {% include footer.html %} - {% include blog-twitter-js.html %} - </body> -</html> diff --git a/site/bazel-release.pub.gpg b/site/bazel-release.pub.gpg deleted file mode 100644 index b29f95fa8f..0000000000 --- a/site/bazel-release.pub.gpg +++ /dev/null @@ -1,53 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mQINBFdEmzkBEACzj8tMYUau9oFZWNDytcQWazEO6LrTTtdQ98d3JcnVyrpT16yg -I/QfGXA8LuDdKYpUDNjehLtBL3IZp4xe375Jh8v2IA2iQ5RXGN+lgKJ6rNwm15Kr -qYeCZlU9uQVpZuhKLXsWK6PleyQHjslNUN/HtykIlmMz4Nnl3orT7lMI5rsGCmk0 -1Kth0DFh8SD9Vn2G4huddwxM8/tYj1QmWPCTgybATNuZ0L60INH8v6+J2jJzViVc -NRnR7mpouGmRy/rcr6eY9QieOwDou116TrVRFfcBRhocCI5b6uCRuhaqZ6Qs28Bx -4t5JVksXJ7fJoTy2B2s/rPx/8j4MDVEdU8b686ZDHbKYjaYBYEfBqePXScp8ndul -XWwS2lcedPihOUl6oQQYy59inWIpxi0agm0MXJAF1Bc3ToSQdHw/p0Y21kYxE2pg -EaUeElVccec5poAaHSPprUeej9bD9oIC4sMCsLs7eCQx2iP+cR7CItz6GQtuZrvS -PnKju1SKl5iwzfDQGpi6u6UAMFmc53EaH05naYDAigCueZ+/2rIaY358bECK6/VR -kyrBqpeq6VkWUeOkt03VqoPzrw4gEzRvfRtLj+D2j/pZCH3vyMYHzbaaXBv6AT0e -RmgtGo9I9BYqKSWlGEF0D+CQ3uZfOyovvrbYqNaHynFBtrx/ZkM82gMA5QARAQAB -tEdCYXplbCBEZXZlbG9wZXIgKEJhemVsIEFQVCByZXBvc2l0b3J5IGtleSkgPGJh -emVsLWRldkBnb29nbGVncm91cHMuY29tPokCPgQTAQIAKAUCV0SbOQIbAwUJA8Jn -AAYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQPVkZtEhFfuA+BA/9ErVPkCbr -bwGX6HHY2h9RULnOuBOJKhqDdlAmJ9Bi1TwliG72rXO52mO7WbSJ4Ux+5bbO3kiz -UId6xr6FZmTJiHCY42gwgLOHLIwd6Y7l+2PTyQ0ryL35hiPvlwhcJ4QzfF0/8MTD -vyQaieBKQQkZD+RmXJLmDGmztLhaUw0BTkONfbN0jTdIt7lAmkVZYo8JCoMbCXlk -9kkh6lx9vXZ0JQkfm1gRRWAAShsOHMYZgXsj11Y3C76zxv8jNoT8315Q4bVMKmjr -+bOaR7idiTlxRKl17eRr8qZ1AqJrwywyjFTWQpwHw3uR4iMt+QSgceKiuihpZBpv -mVtNVaMiJTkR63pRmwaGoGov4u3VMBRa6YMQaXQ1UEwrqEH+cyNZgWidlIywpgzD -4hjLYuE/dCvCVT+M32vTQVhNDCwZkmiNeVQE1R/5AdhQLSvBnLGXIfkXnsCl0vFl -oG2brgxIs3TN8oEitIeVWJ+oj5AqTRBeHIZ5MHktF/18AfPNER4UtufGxUWbAtyf -3HtmZZ03XMct52qSaKZbTY1bOmsgy6ba3Kkc9ifjYa9w9FEVysUPmLzy4+l3K5Xw -92iVYLDtCTThhYfbTJ8IWvJGE9hkyYmPF4Q0O3kFQqQmrFVADMyxR0DMIULs8U3k -bKa5K3939aSYXQAdpq6d6T0y2+XdvSHrmVu5Ag0EV0SbOQEQAOef9VQZQ6VfxJVM -i5kcjws/1fprB3Yp8sODL+QyULqbmcJTMr8Tz83OxprCH5Nc7jsw1oqzbNtq+N2p -OnbAL6XFPolQYuOjKlHGzbQvpH8ZSok6AzwrPNq3XwoB0+12A86wlpajUPfvgajN -jmESMchLnIs3qH1j5ayVICr7vH1i1Wem2J+C/z6gIaG4bko0XKAeU6fNYRmuHLHC -iBiKocpn54LmmPL4ifN7Rz1KkCaAKTT8vKtaVh0g1eswb+9W3qldm+nAc6e1ajWD -iLqhOmTQRVrght80XPYmtv2x8cdkxgECbT6T84rZtMZAdxhjdOmJ50ghPn9o/uxd -CDurhZUsu4aND6EhWw4EfdZCSt0tGQWceB9tXCKVlgc3/TXdTOB9zuyoZxkmQ6uv -rV2ffxf2VLwmR6UJSXsAz2Pd9eWJmnH+QmZPMXhOVFCMRTHTsRfAeyLW+q2xVr/r -c1nV/9PzPP29GSYVb54Fs7of2oHUuBOWp3+2oRljPeoz0SEBG/Q0TdmBqfYTol9r -GapIcROc1qg9oHV6dmQMTAkx3+Io8zlbDp3Xu2+QagtCS+94DcH9Yjh8ggM6hohX -2ofP6HQUw4TLHVTLI0iMc3MJcEZ88voQbHWKT9fYniQjKBESU21IErKT3YWP2OAo -c5RR44gCmE+r14mHCktOLLQrR6sBABEBAAGJAiUEGAECAA8FAldEmzkCGwwFCQPC -ZwAACgkQPVkZtEhFfuDc6g/+PFjc5b156juNTuSyU08/lP58vOWBeL9YxAFcRKVq -mqFQUZSs5xkEA0j0PuueHdPAmLPXO0qE3TRHHcMojI6bpnqPVR6bBHKSE+OUk2yZ -q0LRfsn/Qmn2waIDOKOxA82MAAETiG7/E+obUVGAN2E1fZ30A45eGgqcfr85QYWn -oAWhsenE4gO54ltA8YWyjYvAn+XZ7IegRQE8/Tn2sbKkg8wq/9xnX5rzwVFwanB9 -Oki7djcTKeLtt9xBV6l00icB1bYAPiHGXJiBQElDXiUGqyS/yzCL0El6mbNY0NlO -kSWtfu2Rtg5pofJqs4MA8bF84//9JCHRDpQ35uBtfjhlDP0oCA1uKGM85MbBzpv/ -Be12Fewqy+ofTLZZk1xPUWIUmRggw0cV2PrJpCk0Dg0E4odRI0niY4OjuA8+yiSS -AJAgYs/fSKLqbF7XMLo9yJ56Z/jrDLqO3mYUhcRAg7EiopE/VdEbFppIIu5rZ1hO -PgzFIwsuI+wrcVI1oYYbVD3gIUdJhe8QSWJ7+x1o5vQRJk0vNwh6eBJwCCeQOKn6 -ikXKkkQbc3vsff7wLByyyIKs6zR8aVJk3AHs1QNBusqGZTmBRCrNHg9OJOWfrXSL -WL5zxHbfX+wspgLDLIPYFKlhSEqnfbhrK/17GAd/YF9O7TFy6FzLprmJgp9TaGao -iSI= -=i9Ui ------END PGP PUBLIC KEY BLOCK----- diff --git a/site/blog/_posts/2015-04-06-Simplified-Workspace-Creation.md b/site/blog/_posts/2015-04-06-Simplified-Workspace-Creation.md deleted file mode 100644 index ea5a67b560..0000000000 --- a/site/blog/_posts/2015-04-06-Simplified-Workspace-Creation.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -layout: posts -title: Announcing simplified workspace creation ---- - -To create a new workspace, you can now simply create an empty `WORKSPACE` file -in a directory. - -Previously, you'd need to copy or symlink the `tools` directory into your -project, which was unpopular: - - -<img src="{{ site_root }}/assets/ctrl-w-tweet.png" alt="'move my-project/ to be a subdirectory of base_workspace/' Ok. Ctrl-W." class="img-responsive"> - -[Miguel Alcon](https://github.com/mikelalcon) came up with a great idea for -making this process simpler. Now the `compile.sh` script will create a -`.bazelrc` file in your home directory which tells Bazel where `compile.sh` was -run from and, thus, where it can find its tools when you build. - -To use this new functionality, get the latest version of the code from Github, -run `./compile.sh`, and then create a Bazel workspace by running -`touch WORKSPACE` in any directory. - -Some caveats to watch out for: - -* If you move the directory where Bazel was built you will need to -update your `~/.bazelrc` file. -* If you would like to use different tools than the ones `compile.sh` -finds/generates, you can create a `tools/` directory in your project and -Bazel will attempt to use that instead of the system-wide one. - -See the [getting started]({{ site_root }}/docs/getting-started.html) docs for more info about -setting up your workspace. - -Let us know if you have any questions or issues on the -[mailing list](https://groups.google.com/forum/#!forum/bazel-discuss) or -[GitHub](https://github.com/bazelbuild/bazel). diff --git a/site/blog/_posts/2015-04-10-bash-completion.md b/site/blog/_posts/2015-04-10-bash-completion.md deleted file mode 100644 index 5cd6b1e08e..0000000000 --- a/site/blog/_posts/2015-04-10-bash-completion.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -layout: posts -title: Support for Bash Shell Completion ---- - -We just pushed a support for [shell completion in the Bourne-Again -Shell](https://www.gnu.org/software/bash/manual/html_node/Programmable-Completion.html). -It eases the use of Bazel by expanding its commands and the targets to build. - -To use this new functionality, build the `//scripts:bash_completion` target -from the Bazel repository: -``` -bazel build //scripts:bash_completion -``` - -This will create a `bazel-bin/scripts/bazel-complete.bash` completion script. -You can copy then copy this script to your completion directory -(`/etc/bash_completion.d` in Ubuntu). If you don't want to install it globally -or don't have such a directory, simply add the following line to your -`~/.bashrc` or `~/.bash_profile` (the latter is the recommended for OS X): -``` -source /path/to/bazel/bazel-bin/scripts/bazel-complete.bash -``` - -After that you should be able to type the tab key after the `bazel` -command in your shell and see the list of possible completions. - -If you are interested in supporting other shells, the script is made up -of two parts: - -1. [`scripts/bazel-complete-header.bash`](https://github.com/bazelbuild/bazel/blob/master/scripts/bazel-complete-template.bash) - is the completion logic. -2. `bazel help completion` dumps the list of commands of Bazel, their options - and for commands and options that expect a value, a description of what is - expected. This description is either: - -* an enum of values enclosed into brackets, e.g., `{a,b,c}`; -* a type description, currently one of: - - * `label`, `label-bin`, `label-test`, `label-package` for - a Bazel label for, respectively, a target, a runnable target, - a test, and a package, - * `path` for a filesystem path, - * `info-key` for one of the information keys as listed by `bazel info`; - -* a combination of possible values using `|` as a separator, e.g, - `path|{or,an,enum}'`. - -Let us know if you have any questions or issues on the -[mailing list](https://groups.google.com/forum/#!forum/bazel-discuss) or -[GitHub](https://github.com/bazelbuild/bazel). diff --git a/site/blog/_posts/2015-04-15-share-your-project.md b/site/blog/_posts/2015-04-15-share-your-project.md deleted file mode 100644 index 49ffa2f5db..0000000000 --- a/site/blog/_posts/2015-04-15-share-your-project.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -layout: posts -title: Tell us about your Bazel project! ---- - -We're setting up a list of projects using Bazel. If you'd like us -to list your project on [bazel.build](http://bazel.build), -[send us](https://groups.google.com/d/msg/bazel-discuss/5XaCYQOdN8o/DHE9H4ygHiYJ) -the following information: - -1. The project's name. -2. The language(s) it's using. -3. Whether it uses Bazel + another build system or just Bazel. -4. Any nice surprises/blockers you've run into using Bazel. -5. Any other info or comments you have! - -If you don't want your project publicly listed, we'd still love to hear about -it. Please [email us](mailto:kchodorow@google.com) directly and let us know. diff --git a/site/blog/_posts/2015-04-22-thank-you-stickers.md b/site/blog/_posts/2015-04-22-thank-you-stickers.md deleted file mode 100644 index a94c958811..0000000000 --- a/site/blog/_posts/2015-04-22-thank-you-stickers.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -layout: posts -title: Stickers for Contributors ---- - -<img src="/assets/bazel-stickers.jpg" alt="Bazel stickers" class="img-responsive"> - -We just got Bazel stickers and we'd like to send them to all of the people who -have sent us pull requests and patches over the last month. If you'd like -some stickers, please [send us](mailto:kchodorow@google.com?subject=Send me stickers!) -your Github username and mailing address. - -Let us know if you've done any of the following and we'll send you stickers: - -* Gone through a Gerrit code review. -* Opened a pull request on GitHub. -* Sent us a patch on the mailing list. -* Are in the process of doing any of the things above. - -Thanks for your contributions, we really appreciate them. diff --git a/site/blog/_posts/2015-06-17-visualize-your-build.md b/site/blog/_posts/2015-06-17-visualize-your-build.md deleted file mode 100644 index fc37e879d0..0000000000 --- a/site/blog/_posts/2015-06-17-visualize-your-build.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -layout: posts -title: Visualize your build ---- - -_Reposted from -[Kristina Chodorow's blog](http://www.kchodorow.com/blog/2015/04/24/have-you-ever-looked-at-your-build-i-mean-really-looked-at-your-build/)._ - -Bazel lets you see a graph of your build dependencies. It _could_ help you -debug things, but honestly it's just really cool to see what your build is doing. - -To try it out, you'll need a project that uses Bazel to build. If you don't -have one handy, -[here's a tiny workspace](https://github.com/kchodorow/tiny-workspace) you can -use: - -```bash -$ git clone https://github.com/kchodorow/tiny-workspace.git -$ cd tiny-workspace -``` - -Make sure you've -[downloaded and installed Bazel](http://bazel.build/docs/install.html) and have the -following line to your _~/.bazelrc_: - -``` -query --package_path %workspace%:[path to bazel]/base_workspace -``` - -Now run `bazel query` in your _tiny-workspace/_ directory, asking it to search -for all dependencies of `//:main` and format the output as a graph: - -```bash -$ bazel query 'deps(//:main)' --output graph > graph.in -``` - -This creates a file called _graph.in_, which is a text representation of the -build graph. You can use `dot` (install with `sudo apt-get install graphviz`) -to create a png from this: - -```bash -$ dot -Tpng < graph.in > graph.png -``` - -If you open up _graph.png_, you should see something like this: - -<img src="/assets/graph.png"> - -You can see `//:main` depends on one file (`//:main.cc`) and four targets -(`//:x`, `//tools/cpp:stl`, `//tools/default:crosstool`, and -`//tools/cpp:malloc`). All of the `//tools` targets are implicit dependencies -of any C++ target: every C++ build you do needs the right compiler, flags, and -libraries available, but it crowds your result graph. You can exclude these -implicit dependencies by removing them from your query results: - -```bash -$ bazel query --noimplicit_deps 'deps(//:main)' --output graph > simplified_graph.in -``` - -Now the resulting graph is just: - -<img src="/assets/simple-graph.png"> - -Much neater! - -If you're interested in further refining your query, check out the -[docs on querying](/docs/query.html). diff --git a/site/blog/_posts/2015-06-25-ErrorProne.md b/site/blog/_posts/2015-06-25-ErrorProne.md deleted file mode 100644 index dc4803bd93..0000000000 --- a/site/blog/_posts/2015-06-25-ErrorProne.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -layout: posts -title: Checking your Java errors with Error Prone. ---- - -We recently open-sourced our support for [Error Prone](http://errorprone.info). -[Error Prone](http://errorprone.info) checks for common mistakes in Java code -that will not be caught by the compiler. - -We turned [Error Prone](http://errorprone.info) on by default but you can -easily turn it off by using the Javac option `--extra_checks:off`. To do so, -simply specify `--javacopt='-extra_checks:off'` to the list of Bazel's options. -You can also tune the checks error-prone will perform by using the -[`-Xep:` flags](http://errorprone.info/docs/flags). - -See the [documentation of Error Prone](http://errorprone.info/docs/installation) for more -on Error Prone. diff --git a/site/blog/_posts/2015-07-01-Configuration-File.md b/site/blog/_posts/2015-07-01-Configuration-File.md deleted file mode 100644 index 93b4fc86c1..0000000000 --- a/site/blog/_posts/2015-07-01-Configuration-File.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -layout: posts -title: Sharing your rc files ---- - -You can customize the options Bazel runs with in your `~/.bazelrc`, but -that doesn't scale when you share your workspace with others. - -For instance, you could de-activate [Error Prone](http://errorprone.info)'s -[DepAnn](http://errorprone.info/bugpattern/DepAnn) checks by adding the -`--javacopt="-Xep:DepAnn:OFF"` flag in your `~/.bazelrc`. However, `~/.bazelrc` -is not really convenient as it a user file, not shared with -your team. You could instead add a rc file at `tools/bazel.rc` in your workspace -with the content of the bazelrc file you want to share with your team: - -``` -build --javacopt="-Xep:DepAnn:OFF" -``` - -This file, called a master rc file, is parsed before the user rc file. There is -three paths to master rc files that are read in the following order: - - 1. `tools/bazel.rc` (depot master rc file), - 2. `/path/to/bazel.bazelrc` (alongside bazel rc file), and - 3. `/etc/bazel.bazelrc` (system-wide bazel rc file). - -The complete documentation on rc file is [here](http://bazel.build/docs/bazel-user-manual.html#bazelrc). diff --git a/site/blog/_posts/2015-07-08-Java-Configuration.md b/site/blog/_posts/2015-07-08-Java-Configuration.md deleted file mode 100644 index f9beef32e7..0000000000 --- a/site/blog/_posts/2015-07-08-Java-Configuration.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -layout: posts -title: Configuring your Java builds ---- - -Let say that you want to build for Java 8 and errorprone checks off but -keep the tools directory provided with Bazel in the package path, you could do -that by having the following rc file: - -``` -build --javacopt="-extra_checks:off" -build --javacopt="-source 8" -build --javacopt="-target 8" -``` - -However, the file would becomes quickly overloaded, especially if you take -all languages and options into account. Instead, you can tweak the -[java_toolchain](https://github.com/bazelbuild/bazel/tree/0e1680e58f01f3d443f7e68865b5a56b76c9dadf/tools/jdk/BUILD#L73) -rule that specifies the various options for the java compiler. So in a -BUILD file: - -```python -java_toolchain( - name = "my_toolchain", - encoding = "UTF-8", - source_version = "8", - target_version = "8", - misc = [ - "-extra_checks:on", - ], -) -``` - -And to keep it out of the tools directory (or you need to copy the rest -of the package), you can redirect the default one in a bazelrc: - -``` -build --java_toolchain=//package:my_toolchain -``` - -In the future, toolchain rules should be the configuration points for all -the languages but it is a long road. We also want to make it easier to -rebind the toolchain using the `bind` rule in the WORKSPACE file. - diff --git a/site/blog/_posts/2015-07-23-tree-trimming.md b/site/blog/_posts/2015-07-23-tree-trimming.md deleted file mode 100644 index 203785aa35..0000000000 --- a/site/blog/_posts/2015-07-23-tree-trimming.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -layout: posts -title: Trimming your (build) tree ---- - -_Reposted from [@kchodorow's blog](http://www.kchodorow.com/blog/2015/07/23/trimming-the-build-tree-with-bazel/)._ - -[Jonathan Lange](https://twitter.com/mumak) wrote a [great blog -post](https://jml.io/2015/07/bazel-correct-reproducible-fast-builds.html) about -how Bazel caches tests. Basically: if you run a test, change your code, then run -a test again, the test will only be rerun if you changed something that could -actually change the outcome of the test. Bazel takes this concept pretty far to -minimize the work your build needs to do, in some ways that aren't immediately -obvious. - -Let's take an example. Say you're using Bazel to "build" rigatoni arrabiata, -which could be represented as having the following dependencies: - -<img src="/assets/recipe.png"/> - -Each food is a library which depends on the libraries below it. Suppose you -change a dependency, like the garlic: - -<img src="/assets/change-garlic.png"/> - -Bazel will stat the files of the "garlic" library and notice this change, and -then make a note that the things that depend on "garlic" may have also changed: - -<img src="/assets/dirty.png"/> - -The fancy term for this is "invalidating the upward transitive closure" of the -build graph, aka "everything that depends on a thing might be dirty." Note that -Bazel already knows that this change doesn't affect several of the libraries -(rigatoni, tomato-puree, and red-pepper), so they definitely don't have to be -rebuilt. - -Bazel will then evaluate the "sauce" node and figures out if its output has -changed. This is where the secret sauce (ha!) happens: if the output of the -"sauce" node hasn't changed, Bazel knows that it doesn't have to recompile -rigatoni-arrabiata (the top node), because none of its direct dependencies -changed! - -<img src="/assets/dirty-unmark.png"/> - -The sauce node is no longer “maybe dirty” and so its reverse dependencies -(rigatoni-arrabiata) can also be marked as clean. - -In general, of course, changing the code for a library will change its compiled -form, so the "maybe dirty" node will end up being marked as "yes, dirty" and -re-evaluated (and so on up the tree). However, Bazel's build graph lets you -compile the bare minimum for a well-structured library, and in some cases avoid -compilations altogether. diff --git a/site/blog/_posts/2015-07-28-docker_build.md b/site/blog/_posts/2015-07-28-docker_build.md deleted file mode 100644 index 121261f58f..0000000000 --- a/site/blog/_posts/2015-07-28-docker_build.md +++ /dev/null @@ -1,67 +0,0 @@ ---- -layout: posts -title: Building deterministic Docker images with Bazel ---- - -[Docker](https://www.docker.com) images are great to automate your deployment -environment. By composing base images, you can create an (almost) reproducible -environment and, using an appropriate cloud service, easily deploy those -image. However, V1 Docker build suffers several issues: - - 1. Docker images are non-hermetic as they can run any command, - 2. Docker images are non-reproducible: each "layer" identifier is a **random** - hex string (and not cryptographic hash of the layer content), and - 3. Docker image builds are not incremental since Docker assumes that `RUN foo` - always does the same thing. - -Googlers working on [Google Container Registry](https://gcr.io) developed a support -for building reproducible Docker images using Skylark / Bazel that address these -problems. We recently [shipped](https://github.com/bazelbuild/bazel/commit/5f25891bb17d19cb1208ddad1e88cc4bb4a56782) -it. - -Of course, it does not support `RUN` command, but the rule also strips -timestamps of the tar file and use a SHA sum that is function of the layer -data as layer identifier. This ensure reproducibility and correct -incrementality. - -To use it, simply creates your images using the BUILD language: - -```python -load("/tools/build_defs/docker/docker", "docker_build") - -docker_build( - name = "foo", - tars = [ "base.tar" ], -) - -docker_build( - name = "bar", - base = ":foo", - debs = [ "blah.deb" ], - files = [ ":bazinga" ], - volumes = [ "/asdf" ], -) -``` - -This will generate two docker images loadable with `bazel run :foo` and `bazel -run :bar`. The `foo` target is roughly equivalent to the following Dockerfile: - -``` -FROM bazel/base -``` - -And the `bar` target is roughly equivalent to the following Dockerfile: - -``` -FROM bazel/foo -RUN dpkg -i blah.deb -ADD bazinga / -VOLUMES /asdf -``` - -Using [remote repositories](http://bazel.build/docs/external.html), it is possible -to fetch the various base image for the web and we are working on providing a -`docker_pull` rule to interact more fluently with existing images. - -You can learn more about this docker support -[here](https://github.com/bazelbuild/bazel/blob/master/tools/build_defs/docker/README.md). diff --git a/site/blog/_posts/2015-07-29-dashboard-dogfood.md b/site/blog/_posts/2015-07-29-dashboard-dogfood.md deleted file mode 100644 index 550266b750..0000000000 --- a/site/blog/_posts/2015-07-29-dashboard-dogfood.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -layout: posts -title: Build dashboard dogfood ---- - -__WARNING__: This feature has been removed (2017-04-19). - -We've added a basic dashboard where you can see and share build and test -results. It's not ready for an official release yet, but if any adventurous -people would like to try it out (and please report any issues you find!), feel -free to give it a try. - -<img src="/assets/dash.png" class="img-responsive" style="width: 800px; border: 1px solid black;"/> - -First, you'll need to download or clone [the dashboard project](https://github.com/bazelbuild/dash). - -Run `bazel build :dash && bazel-bin/dash` and add -this line to your `~/.bazelrc`: - -``` -build --use_dash --dash_url=http://localhost:8080 -``` - -Note that the `bazel build` will take a long time to build the first time (the -dashboard uses the AppEngine SDK, which is ~160MB and has to be downloaded). -The "dash" binary starts up a local server that listens on 8080. - -With `--use_dash` specified, every build or test will publish info and logs to -http://localhost:8080/ (each build will print a unique URL to visit). - -<img src="/assets/dash-shell.png"/> - -See [the README](https://github.com/bazelbuild/dash/blob/master/README.md) -for documentation. - -This is very much a work in progress. Please let us know if you have any -questions, comments, or feedback. diff --git a/site/blog/_posts/2015-09-01-beta-release.md b/site/blog/_posts/2015-09-01-beta-release.md deleted file mode 100644 index 67bbcf2040..0000000000 --- a/site/blog/_posts/2015-09-01-beta-release.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -layout: posts -title: Bazel Builder Blasts Beyond Beta Barrier ---- - -_Reposted from [Google's Open Source blog](http://google-opensource.blogspot.com/2015/09/building-build-system-bazel-reaches-beta.html)._ - -We're excited to announce the Beta release of [Bazel](http://bazel.build), an [open -source](https://github.com/bazelbuild/bazel) build system designed to support a -wide variety of different programming languages and platforms. - -There are lots of other build systems out there -- Maven, Gradle, Ant, Make, and -CMake just to name a few. So what's special about Bazel? Bazel is what we use to -build the large majority of software within Google. As such, it has been -designed to handle build problems specific to Google's development environment, -including a massive, shared code repository in which all software is built from -source, a heavy emphasis on automated testing and release processes, and -language and platform diversity. Bazel isn't right for every use case, but we -believe that we're not the only ones facing these kinds of problems and we want -to contribute what we've learned so far to the larger developer community. - -Our beta release provides: - -* Binary releases for - [Linux and OS X](https://github.com/bazelbuild/bazel/releases). -* Support for building and testing C++, Java, Python, Rust, - [and more](http://bazel.build/docs/be/overview.html). -* Support for building Docker images, Android apps, and iOS apps. -* Support for using libraries from - [Maven, GitHub, and more](http://bazel.build/docs/external.html). -* [A robust API](http://bazel.build/docs/skylark/index.html) for adding your own - build rules. - -Check out the [tutorial app](http://bazel.build/docs/tutorial/index.html) to see a -working example using several languages. - -We still have a long way to go. Looking ahead towards our 1.0.0 release, we -plan to provide Windows support, distributed caching, and Go support among other -features. See our [roadmap](http://bazel.build/roadmap.html) for more details -and follow our [blog](http://bazel.build/blog) or -[Twitter](https://twitter.com/bazelbuild) account for regular updates. Feel -free to contact us with questions or feedback on the -[mailing list](https://groups.google.com/forum/#!forum/bazel-discuss) or IRC -(#bazel on freenode). - -_By Jeff Cox, Bazel team_ diff --git a/site/blog/_posts/2015-09-11-sandboxing.md b/site/blog/_posts/2015-09-11-sandboxing.md deleted file mode 100644 index f4c417aed6..0000000000 --- a/site/blog/_posts/2015-09-11-sandboxing.md +++ /dev/null @@ -1,84 +0,0 @@ ---- -layout: posts -title: About Sandboxing ---- - -We've only added sandboxing to Bazel two weeks ago, and we've already seen a -flurry of fixes to almost all of the rules to conform with the additional -restrictions imposed by it. - -## What is sandboxing? -Sandboxing is the technique of restricting the access rights of a process. In -the context of Bazel, we're mostly concerned with restricting file system -access. More specifically, Bazel's file system sandbox contains only known -inputs, such that compilers and other tools can't even see files they should -not access. - -(We currently also mount a number of system directories into the sandbox to -allow running locally installed tools and make it easier to write shell -scripts. See below.) - - -## Why are we sandboxing in Bazel? -We believe that developers should never have to worry about correctness, and -that every build should result in the same output, regardless of the current -state of the output tree. If a compiler or tool reads a file without Bazel -knowing it, then Bazel won't rerun the action if that file has changed, leading -to incorrect incremental builds. - -We would also like to support remote caching in Bazel, where incorrect reuse of -cache entries is even more of a problem than on the local machine. A bad cache -entry in a shared cache affects every developer on the project, and the -equivalent of 'bazel clean', namely wiping the entire remote cache, rather -defeats the purpose. - -In addition, sandboxing is closely related to remote execution. If the build -works well with sandboxing, then it will likely work well with remote -execution - if we know all the inputs, we can just as well upload them to a -remote machine. Uploading all files (including local tools) can significantly -reduce maintenance costs for compile clusters compared to having to install the -tools on every machine in the cluster every time you want to try out a new -compiler or make a change to an existing tool. - - -## How does it work? -On Linux, we're using user namespaces, which are available in Linux 3.8 and -later. Specifically, we create a new mount namespace. We create a temporary -directory into which we mount all the files that the subprocess is allowed to -see. We then use `pivot_root` to make the temporary directory appear as the -root directory for all subprocesses. - -We also mount `/proc`, `/dev/null`, `/dev/zero`, and a temporary filesystem -(tmpfs) on `/tmp`. We mount `/dev/random` and `/dev/urandom`, but recommend -against their usage, as it can lead to non-reproducible builds. - -We currently also mount `/bin`, `/etc`, `/usr` (except `/usr/local`), and every -directory starting with `/lib`, to allow running local tools. In the future, we -are planning to provide a shell with a set of Linux utilities, and to require -that all other tools are specified as inputs. - - -## What about Mac and Windows? -We are planning to implement sandboxing for OS X (using OS X sandboxing, see -our [roadmap](/roadmap.html)) and eventually Windows as well. - - -## What about networking? -At some point, we'd like to also reduce network access, probably also using -namespaces, with a separate opt-out mechanism. - - -## How do I opt-out of sandboxing? -Preferably, you should make all your rules and scripts work properly with -sandboxing. If you need to opt out, you should talk to us first - at Google, -the vast majority of actions is fully sandboxed, so we have some experience -with how to make it work. For example, Bazel has a special mechanism to add -information about the current user, date, time, or the current source control -revision to generated binaries. - -If you still need to opt out for individual rules, you can add the `local = 1` -attribute to `genrule` or `*_test` calls. - -If you're writing a custom rule in Skylark, then you cannot currently opt out. -Instead, please [file a bug](https://github.com/bazelbuild/bazel/issues) and -we'll help you make it work. diff --git a/site/blog/_posts/2015-12-10-java-workers.md b/site/blog/_posts/2015-12-10-java-workers.md deleted file mode 100644 index cc10efb293..0000000000 --- a/site/blog/_posts/2015-12-10-java-workers.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -layout: posts -title: Persistent Worker Processes for Bazel ---- - -Bazel runs most build actions as a separate process. Many build actions invoke a compiler. However, starting a compiler is often slow: they have to perform some initialization when they start up, read the standard library, header files, low-level libraries, and so on. That’s why some compilers and tools have a persistent mode, e.g. [sjavac](http://openjdk.java.net/jeps/199), [Nailgun](http://martiansoftware.com/nailgun/) and [gcc server](http://per.bothner.com/papers/GccSummit03/gcc-server.pdf). Keeping a single process for longer and passing multiple individual requests to the same server can significantly reduce the amount of duplicate work and cut down on compile times. - -In Bazel, we have recently added experimental support for delegating work to [persistent worker processes](https://github.com/bazelbuild/bazel/tree/master/src/main/java/com/google/devtools/build/lib/worker) that run as child processes of and are managed by Bazel. Our Javac wrapper (called JavaBuilder) is the first compiler that supports running as a worker. - -We’ve tried the persistent JavaBuilder for a variety of builds and are seeing a ~4x improvement in Java build times, as Javac can now benefit from JIT optimizations over multiple runs and we no longer have to start a new JVM for every compile action. For Bazel itself, we saw a reduction in build time for a clean build from ~58s to ~16s (on repeated builds). - -<img src="/assets/fullbuild.png" alt="Full build" class="img-responsive"> -<img src="/assets/incbuild.png" alt="Incremental build" class="img-responsive"> - -If you often build Java code, we’d like you to give it a try. Just pass `--strategy=Javac=worker` to enable it or add `build --strategy=Javac=worker` to the .bazelrc in your home directory or in your workspace. Check the WorkerOptions class for [flags to further tune the workers’ behavior](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/worker/WorkerOptions.java) or run “bazel help” and look for the “Strategy options” category. Let us know how it works for you. - -We’re currently using a simple [protobuf-based protocol](https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/worker_protocol.proto) to communicate with the worker process. Let us know if you want to add support for more compilers; in many cases, you can do that without any Bazel changes. However, the protocol is still subject to change based on your feedback. diff --git a/site/blog/_posts/2015-3-27-Hello-World.md b/site/blog/_posts/2015-3-27-Hello-World.md deleted file mode 100644 index b92055175e..0000000000 --- a/site/blog/_posts/2015-3-27-Hello-World.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -layout: posts -title: Hello World ---- - -Welcome to the Bazel blog! We'll be using this forum for news and -announcements. diff --git a/site/blog/_posts/2016-01-27-continuous-integration.md b/site/blog/_posts/2016-01-27-continuous-integration.md deleted file mode 100644 index b843a17d01..0000000000 --- a/site/blog/_posts/2016-01-27-continuous-integration.md +++ /dev/null @@ -1,158 +0,0 @@ ---- -layout: posts -title: Using Bazel in a continuous integration system ---- - -When doing continuous integration, you do not want your build to fail because a -a tool invoked during the build has been updated or some environmental -conditions have changed. Because Bazel is designed for reproducible builds and -keeps track of almost every dependency of your project, Bazel is a great tool -for use inside a CI system. Bazel also caches results of previous build, -including test results and will not re-run unchanged tests, speeding up each -build. - -## Running Bazel on virtual or physical machines. - -For [ci.bazel.build](http://ci.bazel.build), we use -[Google Compute Engine](https://cloud.google.com/compute/) virtual machine for -our Linux build and a physical Mac mini for our Mac build. Apart from Bazel -tests that are run using the -[`./compile.sh`](https://github.com/bazelbuild/bazel/blob/master/compile.sh) -script, we also run some projects to validate Bazel binaries against: the -[Bazel Tutorial](https://github.com/bazelbuild/examples/tree/master/tutorial) -[here](http://ci.bazel.build/job/Tutorial/), -[re2](https://github.com/google/re2) [here](http://ci.bazel.build/job/re2/), -[protobuf](https://github.com/google/protobuf) -[here](http://ci.bazel.build/job/protobuf/), and -[TensorFlow](https://www.tensorflow.org) -[here](http://ci.bazel.build/job/TensorFlow/). - -Bazel is reinstalled each time we run the tutorial or TensorFlow, but the Bazel -cache is maintained across installs. The setup for those jobs is the following: - -```bash -set -e - -# Fetch the Bazel installer -URL=https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${INSTALLER_PLATFORM}.sh -export BAZEL_INSTALLER=${PWD}/bazel-installer/install.sh -curl -L -o ${BAZEL_INSTALLER} ${URL} -BASE="${PWD}/bazel-install" - -# Install bazel inside ${BASE} -bash "${BAZEL_INSTALLER}" \ - --base="${BASE}" \ - --bazelrc="${BASE}/bin/bazel.bazelrc" \ - --bin="${BASE}/binary" - -# Run the build -BAZEL="${BASE}/binary/bazel --bazelrc=${BASE}/bin/bazel.bazelrc" -${BAZEL} test //... -``` - -This tests installing a specific version of Bazel each time. Of course, if -Bazel is installed on the path, one can simply `bazel test //...`. However, -even with reinstalling all the time, Bazel caching simply works. - - -## Running Bazel inside a Docker container - -Several people want to use Bazel in a Docker container. First of all, Bazel -has some feature that are incompatibles with Docker: - -- Bazel runs by default in client/server mode using UNIX domain sockets, so if - you cannot mount the socket inside the Docker container, then you must disable - client-server communication by running Bazel in batch mode with the `--batch` - flag. -- Bazel [sandboxes all actions on linux by default](http://bazel.build/blog/2015/09/11/sandboxing.html) - and this needs special privileges in the Docker container (enabled by - [`--privilege=true`](https://docs.docker.com/engine/reference/run/#runtime-privilege-linux-capabilities-and-lxc-configuration). - If you cannot enable the namespace sandbox, you can deactivate it in Bazel - with the `--genrule_strategy=standalone --spawn_strategy=standalone` flags. - -So the last step of the previous script would look like: - -```bash -# Run the build -BAZEL="${BASE}/binary/bazel --bazelrc=${BASE}/bin/bazel.bazelrc --batch" -${BAZEL} test --genrule_strategy=standalone --spawn_strategy=standalone \ - //... -``` - -This build will however be slower because the server has to restart for every -build and the cache will be lost when the Docker container is destroyed. - -To prevent the loss of the cache, it is better to mount a persistent volume for -`~/.cache/bazel` (where the Bazel cache is stored). - - -## Return code and XML output - -A final consideration when setting up a continuous integration system is getting -the result from the build. Bazel has the following interesting exit codes when -using `test` and `build` commands: - -<table class="table table-condensed table-striped"> - <thead> - <tr> - <th>Exit Code</th> - <th>Description</th> - </tr> - </thead> - <tbody> - <tr> - <td>0</td> - <td>Success.</td> - </tr> - <tr> - <td>1</td> - <td>Build failed.</td> - </tr> - <tr> - <td>2</td> - <td> - Command Line Problem, Bad or Illegal flags or command combination, or - Bad Environment Variables. Your command line must be modified. - </td> - </tr> - <tr> - <td>3</td> - <td>Build OK, but some tests failed or timed out.</td> - </tr> - <tr> - <td>4</td> - <td> - Build successful but no tests were found even though testing was - requested. - </td> - </tr> - <tr> - <td>8</td> - <td> - Build interrupted (by a <kbd><kbd>Ctrl</kbd>+<kbd>C</kbd></kbd> from the - user for instance) but we terminated with an orderly shutdown. - </td> - </tr> - </tbody> -</table> - -These return codes can be used to determine the reason for a failure -(in [ci.bazel.build](http://ci.bazel.build), we mark builds that have exited with exit -code 3 as unstable, and other non zero code as failed). - -You can also control how much information about test results Bazel prints out -with the [`--test_output` flag](http://bazel.build/docs/bazel-user-manual.html#flag--test_output). -Generally, printing the output of test that fails with `--test_output=errors` is -a good setting for a CI system. - -Finally, Bazel's built-in [JUnit test runner](https://github.com/bazelbuild/bazel/blob/master/src/java_tools/junitrunner) -generates Ant-style XML output file (in `bazel-testlogs/pkg/target/test.xml`) -that summarizes the results of your tests. This test runner can be activated -with the `--nolegacy_bazel_java_test` flag (this will soon be the default). -Other tests also get [a basic XML output file](https://github.com/bazelbuild/bazel/blob/master/tools/test/test-setup.sh#L54) -that contains only the result of the test (success or failure). - -To get your test results, you can also use the -[Bazel dashboard](http://bazel.build/blog/2015/07/29/dashboard-dogfood.html), -an optional system that automatically uploads Bazel test results to a shared -server. diff --git a/site/blog/_posts/2016-02-23-0.2.0-release.md b/site/blog/_posts/2016-02-23-0.2.0-release.md deleted file mode 100644 index bc40805b3c..0000000000 --- a/site/blog/_posts/2016-02-23-0.2.0-release.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -layout: posts -title: Bazel 0.2.0 Released ---- - -We are delighted to announce the -[0.2.0 release](https://github.com/bazelbuild/bazel/releases/tag/0.2.0) of -Bazel. This release marks major improvements in support for external -repositories, Skylark and testing, in particular how external repositories and -Skylark can work together. - -## Improvements from our [roadmap](http://bazel.build/roadmap.html) - -### Skylark rules can now be loaded from remote repositories - -Skylark rules can now be loaded from a remote repository. For example, to use -the Scala rules, add the following to your WORKSPACE file: - -```python -git_repository( - name = "io_bazel_rules_scala", - remote = "https://github.com/bazelbuild/rules_scala.git", - tag = "0.0.1", -) -load("@io_bazel_rules_scala//scala:scala.bzl", "scala_repositories") -scala_repositories() -``` - -This will download all of the tools the rules need to build Scala programs. - -Then load and use normally from your BUILD files: - -```python -load("@io_bazel_rules_scala//scala:scala.bzl", "scala_library") -scala_library(...) -``` - -We will gradually move the existing rules to their own repositories, announcing -changes on the [mailing -list](https://groups.google.com/forum/#!forum/bazel-discuss). - -### Go build and test support - -There is now Go language support, see [the -documentation](http://bazel.build/docs/be/go.html) for details. - -### Open sourcing tests - -We also open sourced over a -[hundred](https://github.com/bazelbuild/bazel/tree/master/src/test/java/com/google/devtools) -[tests](https://github.com/bazelbuild/bazel/tree/master/src/test/shell/integration) -and laid the foundation for open sourcing more. We will continue to open source -more tests (both to increase Bazel's stability and to make contributing easier), -but this marks a dramatic increase in test coverage. - -## Other major changes since 0.1.0 - -* The `--package_path` definition in `.bazelrc` is no longer required, nor is - the `base_workspace/` directory. -* JUnit test runner for Java tests - Use the `--nolegacy_bazel_java_test` flag - (soon to be the default) to get XML output for easy integration into CI - systems and easier debugging with `--java_debug`. -* Skylark macros can now be loaded and used in the WORKSPACE file. -* Remote repository filesystem changes are tracked. -* [Debian](https://github.com/bazelbuild/bazel/releases/latest/) packages and a - [Homebrew](http://braumeister.org/formula/bazel) recipe. - -For changes since 0.1.5 (the minor version before 0.2.0), see the release notes -for changes. - -## Future plans - -Looking ahead to 0.3.0: - -* Windows support is coming! (See the [Windows - label](https://github.com/bazelbuild/bazel/issues?q=is%3Aissue+is%3Aopen+label%3A%22category%3A+multi-platform+%3E+windows%22) to follow the - progress there). -* Remote caching and execution is in progress (see [Alpha Lam's - work](https://docs.google.com/document/d/1CvEw3uu9mUszK-ukmSWp4Dmy43KDlHAjW75Gf17bUY8/preview)). -* XCode integration and generic IDE support. -* [Ulf](https://github.com/ulfjack) has been working on [sandboxing for OS - X](https://github.com/bazelbuild/bazel/tree/osx-sandbox-hardlinks), which will - hopefully be available soon. -* More work on parallelization. We currently have experimental support (which - can be enabled with the `--experimental_interleave_loading_and_analysis` flag) - which improves clean build time (~30% faster loading and analysis), especially - for builds using a lot of `select()` expressions. - -## Finally... - -A big thank you to our community for your continued support. Particular -shout-outs to the following contributors: - -* [Brian Silverman](https://github.com/bsilver8192) - for tons of important bug - fixes and answering lots of user questions. -* [Alpha Lam](https://github.com/hhclam) - for writing up design docs and - implementing remote caching/execution. -* [P. Oscar Boykin](https://github.com/posco) - for putting tons of time and - effort into the Scala rules, as well as being a tireless supporter on Twitter. - -Thank you all, keep the discussion and bug reports coming! diff --git a/site/blog/_posts/2016-03-18-sandbox-easier-debug.md b/site/blog/_posts/2016-03-18-sandbox-easier-debug.md deleted file mode 100644 index 64f6841488..0000000000 --- a/site/blog/_posts/2016-03-18-sandbox-easier-debug.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -layout: posts -title: Easier Debugging of Sandbox Failures ---- - -We have often heard that debugging failed execution due to issues with the sandbox is difficult and requires knowledge of the sandboxing code of Bazel to actually understand what’s happening. With [these changes](https://github.com/bazelbuild/bazel/commit/40ee9de052e3bb8cf5a59eeff3936148e1f55e69), we hope that you will be able to solve common issues easier on your own and make debugging easier. - - -If you don’t know much about Bazel sandbox, you might want to read [this blog post](http://bazel.build/blog/2015/09/11/sandboxing.html) - - -## What we did: - -- When using `--verbose_failures` and `--sandbox_debug`, Bazel now shows the detailed command that it ran when your build failed, including the part that sets up the sandbox. -- When you copy & paste the shown command into a terminal, the failed command is rerun - but when it fails this time, we provide access to a shell inside a new sandbox which is the same as the old sandbox we made before, so that you can explore the sandbox from the inside and find out why the command failed. - -## How to use it: -Let’s say you wrote a Skylark rule and forgot to add your compiler to the input files. Before this change, when you ran bazel build, you would get several error messages like this: - -``` -ERROR: path/to/your/project/BUILD:1:1: compilation of rule '//path/to/your/project:all' failed: No such file or directory. -ERROR: /your/project/BUILD:x:1: Executing genrule //project/dir:genrule failed: bash failed: error executing command /path/to/your/compiler some command -``` - -But you probably have no idea what to do, because the error message is not detailed enough and you have everything you needed in your system. - -With this new feature, you will get an error message like this instead: - -``` -ERROR: path/to/your/project/BUILD:1:1: compilation of rule '//path/to/your/project:all' failed: - -Sandboxed execution failed, which may be legitimate (e.g. a compiler error), or due to missing dependencies. To enter the sandbox environment for easier debugging, run the following command in parentheses. On command failure, a bash shell running inside the sandbox will then automatically be spawned - -namespace-sandbox failed: error executing command - (cd /some/path && \ - exec env - \ - LANG=en_US \ - PATH=/some/path/bin:/bin:/usr/bin \ - PYTHONPATH=/usr/local/some/path \ - /some/path/namespace-sandbox @/sandbox/root/path/this-sandbox-name.params -- /some/path/to/your/some-compiler --some-params some-target) -``` - -Then you can simply copy & paste the command above in parentheses into a new terminal: - -``` -(cd /some/path && \ - exec env - \ - LANG=en_US \ - PATH=/some/path/bin:/bin:/usr/bin \ - PYTHONPATH=/usr/local/some/path \ - /some/path/namespace-sandbox @/sandbox/root/path/this-sandbox-name.params -- /some/path/to/your/some-compiler --some-params some-target) -``` - -There will be the same error message about not finding your compiler, but after that error message, you will find yourself in a bash shell inside a new sandbox. You can now debug the failure, e.g. you can explore the sandbox: look for any missing file, check for possible errors in your BUILD files, run your compiler again manually, or even use strace. - -For this example, we run our compiler in the sandbox again manually and the error message shows `No command ‘some-compiler’ found` - looking around, you notice that the compiler binary is missing. This means it was not part of the action inputs, because Bazel always mounts all action inputs into the sandbox - so you check out your Skylark rule and notice that this is indeed the case. Adding your compiler to the input files in your Skylark rule should thus fix the error. - -Next time you run bazel build, it should mount your compiler into the sandbox and thus find it correctly. If you get a different error, you could repeat the steps above. diff --git a/site/blog/_posts/2016-03-31-autoconfiguration.md b/site/blog/_posts/2016-03-31-autoconfiguration.md deleted file mode 100644 index 0821f7bb32..0000000000 --- a/site/blog/_posts/2016-03-31-autoconfiguration.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -layout: posts -title: Using Skylark remote repositories to auto-detect the C++ toolchain. ---- - -[Skylark remote repositories](/docs/skylark/repository_rules.html) let you -create custom [external repositories](/docs/external.html) using -[Skylark](/docs/skylark/index.html). This not only enables creating rules for -custom package systems such as [PyPi](https://pypi.python.org) but also generating -a repository to reflect the toolchain installed on the workstation Bazel is running -on. We explain here how we implemented [auto-configuration for the C++ -toolchain](https://github.com/bazelbuild/bazel/blob/master/tools/cpp/cc_configure.bzl). - -## Principles - -<blockquote> -C++ toolchain: the set of binaries and libraries required to build C++ code. - -Crosstool: a compiler capable of building for a certain architecture, which -can be different from the host architecture (e.g., gcc running on Linux and -building for Raspberry Pi). -</blockquote> - -C++ toolchains are configured in Bazel using a [crosstool target](https://github.com/bazelbuild/bazel/blob/8fa5ae6a6364100f2a7f9130e62eb0edb447339a/tools/cpp/BUILD#L32) -and a [CROSSTOOL file](https://github.com/bazelbuild/bazel/blob/master/tools/cpp/CROSSTOOL). - -This crosstool target (:default_toolchain) is the first step in moving the contents -of the CROSSTOOL file entirely into BUILD file rules. The CROSSTOOL file defines -where to find the C++ compiler, its include directories and also the various flag -to use at each compilation step. - -When your C++ compiler is not in the standard location, then this static -CROSSTOOL file cannot find it. To cope with the variety of installation out -there, we created a `cc_configure` Skylark repository rule that will generates -a `@local_config_cc//tools/cpp` package containing a generated CROSSTOOL file -based on the information we gathered from the operating system. - - -## Implementation - -The [`cc_configure`](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L291) -rule is actually a macro wrapping the [`cc_autoconf`](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L288) -enforcing the `local_config_cc` name for the repository. The -[implementation](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L255) -of the `cc_autoconf` rule does the following step: - - - [Detect the `cpu_value`](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L85) - using the [`repository_ctx.os.name`](/docs/skylark/lib/repository_os.html#name) attribute. - - Generates a [static package](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L85) - if we do not support the target platform. - - Detect the [C++ compiler path](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L235) - using [`repository_ctx.which`](/docs/skylark/lib/repository_ctx.html#which) and the `CC` environment variable with - [`repository_ctx.os.environ`](/docs/skylark/lib/repository_os.html#environ). - - Detect some [more tool paths](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L53), - still using [`repository_ctx.which`](/docs/skylark/lib/repository_ctx.html#which). - - Generates the [various flag for the `CROSSTOOL` file](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L127), - [testing flags against the detected compiler](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L114) - using [`repository_ctx.execute`](/docs/skylark/lib/repository_ctx.html#execute). We also - [detect the include directories](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L101) - with [`repository_ctx.execute`](/docs/skylark/lib/repository_ctx.html#execute). - - With the gathered information, generate the C++ tools package: its [BUILD file](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L274), - [wrapper script for Darwin](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L278) and - [CROSSTOOL file](https://github.com/bazelbuild/bazel/blob/9116b3e99af2fd31d92c9bb7c37905a1675456c1/tools/cpp/cc_configure.bzl#L279) using - [`repository_ctx.template`](/docs/skylark/lib/repository_ctx.html#template). - -So using the function provided by [`repository_ctx`](/docs/skylark/lib/repository_ctx.html), we can discover -the binaries on the system, what version they are, and which options they support, then generate a -configuration to match the local C++ toolchain. - - -## Creating your own repository rules - -When creating a Skylark remote repository, a few things should be taken in considerations: - - - The Skylark implementation of a remote repository is run during the loading phase of - the repository, which means that unless the rule definition is changed in the WORKSPACE - file or the implementation fails, it will not be re-run unless the user does a - `bazel clean --expunge`. We are thinking of further command to force re-run that loading - phase for a specific remote repository ([#974](https://github.com/bazelbuild/bazel/issues/974)). - - Skylark remote repository can do a lot of non hermetic operation, it is recommended - to check as many things as possible to ensure hermeticity (and overall, we recommend - using a vendored toolchain instead of using auto-detected one if reproducibility is important). - For example, it is recommended to use the `sha256` argument of the - [`repository_ctx.download`](/docs/skylark/lib/repository_ctx.html#download) method. - - Naming a rule can be complex and we recommend to not use standard suffix of classical - rules for remote repositories (e.g. `*_library` or `*_binary`). If you create a - package rule, a good name would probably be `xxx_package` (e.g., `pypi_package`). If - you create an autoconfiguration rule, `xxx_configure` is probably the best name - (e.g. `cc_configure`). diff --git a/site/blog/_posts/2016-06-10-0.3.0-release.md b/site/blog/_posts/2016-06-10-0.3.0-release.md deleted file mode 100644 index d8a97c3a71..0000000000 --- a/site/blog/_posts/2016-06-10-0.3.0-release.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -layout: posts -title: Bazel 0.3.0 Released ---- - -We are delighted to announce the -[0.3.0 release](https://github.com/bazelbuild/bazel/releases/tag/0.3.0) of -Bazel. This milestone is marked by support for IDE integration but also major -features such as remote caching of build artifacts and experimental Windows -support. - -## Improvements from our [roadmap](http://bazel.build/roadmap.html) - -### IDE support - -In this release, we made it possible to [generate information for IDEs from -Bazel build files](http://bazel.build/blog/2016/06/10/ide-support.html) using -[Skylark aspects](http://bazel.build/docs/skylark/aspects.html). - -Simultaneous with Bazel 0.3 release, we are announcing the availability of two -projects integrating Bazel with different IDEs: - -* [Tulsi](http://tulsi.bazel.build) is an Xcode plugin for Bazel. This is the same - plugin that teams inside Google use for developing iOS applications. -* [e4b](https://github.com/bazelbuild/e4b) is an experimental Eclipse plugin for - Bazel. It was made to illustrate the use of Skylark aspects for IDE - integration. This is an experimental plugin but we welcome any contributions - to it. - -### Windows support - -Bazel can now bootstrap on [Windows](http://bazel.build/docs/windows.html) without -admin privilege and can use the Microsoft Visual C++ toolchain. Windows -support is still highly experimental and we have identified -[several issues](https://github.com/bazelbuild/bazel/issues?q=is%3Aissue+is%3Aopen+label%3A%22category%3A+multi-platform+%3E+windows%22) -and their solutions. We are dedicated to a good native experience on Windows. - -### Remote caching of distributed artifacts - -[Alpha Lam](https://github.com/hhclam) has contributed experimental support -for [distributed caching](https://github.com/bazelbuild/bazel/commit/79adf59) -and [execution](https://github.com/bazelbuild/bazel/commit/a1a79cb). This is -an ongoing area of development and several engineers from Google are working -with Alpha to enhance that support. - -### Skylark remote repositories - -Remote repository rules can now be created using -[Skylark](http://bazel.build/docs/skylark/repository_rules.html). This can be used -to support your custom protocols, interfacing with new packaging systems or even -do auto-configuration to use a toolchain on your local disk. We use it -especially to have [a better out-of-the-box experience with C++ toolchains](http://www.bazel.build/blog/2016/03/31/autoconfiguration.html). - -## Other major changes since 0.2.0 - -* We just open-sourced our BUILD file formatter, [buildifier](https://github.com/bazelbuild/buildifier). -* We now provide a Debian APT repository for installing bazel, see the - [installation guide](http://bazel.build/docs/install.html) on how to use it. -* Our JUnit test runner for Java tests (`--nolegacy_bazel_java_test`) is now the - default. - -For changes since 0.2.3 (the minor version before 0.3.0), see the -[release notes](https://github.com/bazelbuild/bazel/releases/tag/0.3.0) for -changes. - -## Future plans - -Looking ahead to 0.4.0: - -* The last blockers for `--strategy=worker=Javac` will be resolved, making Java - builds [faster](http://www.bazel.build/blog/2015/12/10/java-workers.html). -* [Yue](https://github.com/hermione521) has made great progress in making - [Ulf](https://github.com/ulfjack)'s prototype of [sandboxing for OS - X](https://github.com/bazelbuild/bazel/tree/osx-sandbox-hardlinks) real. - -## Finally... - -A big thank you to our community for your continued support. Particular -shout-outs to the following contributors: - -* [Justine Tunney](https://github.com/jart) - for developing and maintaining - the [Closure JS rules](https://github.com/bazelbuild/rules_closure) of Bazel. -* [Alpha Lam](https://github.com/hhclam) - for implementing remote - caching/execution and following up on these features. -* [David Chen](https://github.com/davidzchen) - for going above and beyond, far - more than a standard 20% contribution: improving our documentation, - creating the [Skylark documentation generator](https://github.com/bazelbuild/skydoc), - fixing bugs and contributing features in Bazel and helping out TensorFlow with - their use of Bazel. - -Thank you all, keep the discussion and bug reports coming! diff --git a/site/blog/_posts/2016-06-10-ide-support.md b/site/blog/_posts/2016-06-10-ide-support.md deleted file mode 100644 index 34b7d6a6bb..0000000000 --- a/site/blog/_posts/2016-06-10-ide-support.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -layout: posts -title: IDE support ---- - -One of Bazel’s longest-standing feature requests is integration with IDEs. -With the 0.3 release, we finally have all machinery in place that allows -implementing integration with Bazel in IDEs. Simultaneous with that -Bazel release we are also making public two IDE plugins: - -* [Tulsi](http://tulsi.bazel.build): Bazel support for Xcode. -* [e4b](https://github.com/bazelbuild/e4b): a sample Bazel plugin for Eclipse. - -In this post, we will look into how Bazel enables IDE integration -and how an IDE plugin integrating with Bazel can be implemented. - - -## Principles of Bazel IDE support - -Bazel BUILD files provide a description of a project’s source code: what -source files are part of the project, what artifacts (targets) should be -built from those files, what the dependencies between those files are, etc. -Bazel uses this information to perform a build, that is, it figures out the set -of actions needed to produce the artifacts (such as running a compiler or -linker) and executes those actions. Bazel accomplishes this by constructing a -_dependency graph_ between targets and visiting this graph to collect -those actions. - -IDEs (as well as other tools working with source code) also need the same -information about the set of sources and their roles; but instead of building -the artifacts, IDEs use it to provide code navigation, autocompletion and -other code-aware features. - -In the 0.3.0 Bazel release, we are adding a new concept to Bazel - -[_aspects_](/docs/skylark/aspects.html). -Aspects allow augmenting build dependency graphs with additional information -and actions. Applying an aspect to a build target creates a "shadow -dependency graph" reflecting all transitive dependencies of that target, -and the aspect's implementation determines the actions that Bazel executes -while traversing that graph. -The [documentation on aspects](/docs/skylark/aspects.html) explains this in more -detail. - -## Architecture of a Bazel IDE plugin. - -As an example of how aspects are useful for IDE integration, we will take -a look at a sample -[Eclipse plugin for Bazel support, e4b](https://github.com/bazelbuild/e4b). - -e4b includes an aspect, defined in a file -[`e4b_aspect.bzl`](https://github.com/bazelbuild/e4b/blob/master/com.google.devtools.bazel.e4b/resources/tools/must/be/unique/e4b_aspect.bzl), -that when -applied to a particular target, generates a small JSON file with information -about that target relevant to Eclipse. Those JSON files are then consumed -by the e4b plugin inside Eclipse to build [Eclipse's representation -of a project](https://github.com/bazelbuild/e4b/blob/master/com.google.devtools.bazel.e4b/src/com/google/devtools/bazel/e4b/classpath/BazelClasspathContainer.java), -[`IClasspathContainer`](http://help.eclipse.org/juno/index.jsp?topic=%2Forg.eclipse.jdt.doc.isv%2Freference%2Fapi%2Forg%2Feclipse%2Fjdt%2Fcore%2FIClasspathContainer.html): - - - -Through the e4b plugin UI, the user specifies an initial set of targets -(typically a java or android binary, a selection of tests, all targets -in certain packages, etc). E4b plugin then invokes bazel as follows: - -``` -bazel build //java/com/company/example:main \ ---aspects e4b_aspect.bzl%e4b_aspect \ ---output_groups ide-info -``` - -(some details are omitted for clarity; see -[e4b source](https://github.com/bazelbuild/e4b/blob/master/com.google.devtools.bazel.e4b/src/com/google/devtools/bazel/e4b/command/BazelCommand.java) for complete -invocation) - -The `--aspects` flag directs Bazel to apply `e4b_aspect`, exported from -`e4bazel.bzl` Skylark extension, to target `//java/com/company/example:main`. - -The aspect is then applied transitively to the dependencies of the specified -targets, producing `.e4b-build.json` files for each target in the transitive -closure of dependencies. The e4b plugin reads those outputs and provides -a Classpath for Eclipse core to consume. If the input BUILD files change -so that a project model needs to be re-synced, the plugin still invokes -the exact same command: Bazel will rebuild only those files that are affected -by the change, so the plugin need only reexamine only those newly built -`.e4b-build.json` files. `ide-info` is an output group defined by e4b\_aspect; -the `--output_groups` flag ensures that only the artifacts belonging to that -group (and hence only to the aspect) are built, and therefore that no -unnecessary build steps are performed. - -The aspect uses the -[`java` provider](/docs/skylark/lib/JavaSkylarkApiProvider.html) on the targets -it applies to to access a variety of information about Java targets. - - diff --git a/site/blog/_posts/2016-10-07-bazel-windows.md b/site/blog/_posts/2016-10-07-bazel-windows.md deleted file mode 100644 index 30b89ba0a0..0000000000 --- a/site/blog/_posts/2016-10-07-bazel-windows.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -layout: posts -title: Bazel on Windows ---- - -We first announced experimental Windows support in 0.3.0. Since then, we've -implemented support for building, running and testing C++, Java and Python, -as well as improved performance and stability. Starting with -Bazel version 0.3.2, we are making prebuilt Bazel Windows binaries available -as part of our -[releases](https://github.com/bazelbuild/bazel/releases) -([installation instructions](/docs/windows.html#using-the-release-binary)). - -In addition to bootstrapping Bazel itself, we're also able to build -significant parts of TensorFlow with Bazel on Windows -([pull request](https://github.com/tensorflow/tensorflow/pull/4796)). -Bazel on Windows currently requires [msys2](https://msys2.github.io/) and -still has a number of issues. Some of the more important ones are: - - * [Workspace of the project needs to be on C: drive](https://github.com/bazelbuild/bazel/issues/1463) - * [Runfiles will require additional tweaking](https://github.com/bazelbuild/bazel/issues/1212) - * We support [building C++ code with MSVC toolchain](/docs/windows.html#build-c), - but it is not yet the default toolchain. - -Our GitHub issue tracker has a [full list of known issues](https://github.com/bazelbuild/bazel/issues?q=is%3Aissue+is%3Aopen+label%3A%22category%3A+multi-platform+%3E+windows%22). - -Now, we need your help! Please try building your Bazel project on Windows, -and let us know what works or what doesn't work yet, and what we can do better. - -We are looking forward to what you build (on Windows)! - diff --git a/site/blog/_posts/2016-10-20-intellij-support.md b/site/blog/_posts/2016-10-20-intellij-support.md deleted file mode 100644 index 4f95745224..0000000000 --- a/site/blog/_posts/2016-10-20-intellij-support.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -layout: posts -title: IntelliJ and Android Studio support ---- - -We've recently open-sourced Bazel plugins for -[IntelliJ and Android Studio](https://ij.bazel.build). - -## Key Features ## - -* Import a project directly from a BUILD file. -* BUILD file integration: syntax highlighting, refactoring, find usages, - code completion, etc. [Skylark](/docs/skylark) - extensions are fully supported. -* Compile your project and get navigatable Bazel compile errors in the IDE. -* [Buildifier](https://github.com/bazelbuild/buildifier) integration -* Support for Bazel run configurations for certain rule classes. -* Run/debug tests directly through Bazel by right-clicking on - methods/classes/BUILD targets. - -## How do I get started? ## - -To try them out, you can install them directly from within the IDE -(**Settings > Plugins > Browse repositories**), download them from the -JetBrains [plugin repository](https://plugins.jetbrains.com/search/index?search=bazel), -or build directly from [source](https://github.com/bazelbuild/intellij). - -Detailed docs are available [here](https://ij.bazel.build). - -The plugins are currently Linux-only, with plans for Mac support in the future. diff --git a/site/blog/_posts/2016-11-02-0.4.0-release.md b/site/blog/_posts/2016-11-02-0.4.0-release.md deleted file mode 100644 index 810be95535..0000000000 --- a/site/blog/_posts/2016-11-02-0.4.0-release.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -layout: posts -title: Bazel 0.4.0 Released ---- - -We are delighted to announce the [0.4.0 release of Bazel](https://github.com/bazelbuild/bazel/releases/tag/0.4.0). This release marks major improvements in support for Windows, sandboxing and performance. - -We are also moving also moving to a new domain. [https://bazel.build](https://bazel.build) is already up and running and we are slowly moving every reference to point to that new domain.. - -## Improvements from our [roadmap](http://bazel.build/roadmap.html) - -### Java workers are now used by default - -Java workers reduce the time of java compilation by keeping a hot JVM in the background. [It improves java compilation speed by 5x](https://bazel.build/blog/2015/12/10/java-workers.html) and we decided to make it the default. - -### Sandboxing is now working also on OS X - -With our [Beta release](https://bazel.build/blog/2015/09/01/beta-release.html), we added sandboxing of action to Linux. This feature ensured that the rule set does not access undeclared input, allowing correct execution of actions. We leveraged the [sandbox-exec](https://developer.apple.com/legacy/library/documentation/Darwin/Reference/ManPages/man1/sandbox-exec.1.html) command to generate a similar sandbox on OS X. - -## Other major changes since 0.3.0 - -### We provide Bazel binaries for Windows - -As announced in our Bazel on Windows blog post, we are now providing binary distribution of [Bazel for Windows](https://bazel.build/blog/2016/10/07/bazel-windows.html). A [chocolatey package](https://chocolatey.org/packages/bazel) was contributed by [Peter Mounce](https://github.com/petemounce) so you can just do choco install bazel to get Bazel installed on Windows, big thanks to Peter Mounce! This release also marks a big step for us: TensorFlow PIP package can now be built on Windows with Bazel! - -## Skylark implementation of repository rules - -We now have implementation of two repository rules (git_repository and maven_jar) in Skylark and we recommend using them instead of the native ones, to do so simply adds the following lines at the top of your WORKSPACE file: - -```python -load( - "@bazel_tools//tools/build_defs/repo:git.bzl", - "git_repository", - "new_git_repository", -) -load("@bazel_tools//tools/build_defs/repo:maven_rules.bzl", "maven_jar") -``` - -### And various more - -* The --watchfs flag is ready to be turned on by default. It improves performance of Bazel, try it out! -* The Linux sandbox got revamped for better performance and usability: no performance hit should be perceptible and accessing system tools should be possible. - -For changes since 0.3.2 (the minor version before 0.4.0), see the release notes for changes. - -## Future plans - -### Looking ahead to 0.5.0: - -* With the help of your feedback, we will resolve the last issue to make our Windows port work seamlessly for Java, C++ and Python. -* The new distributed execution API will be stabilized. - -## Finally... - -A big thank you to our community for your continued support. Particular shout-outs to the following contributors: - -* [Yugui](https://github.com/yugui) for her contribution to [rules_go](https://github.com/bazelbuild/rules_go). -* [Paul Bethe](https://github.com/pmbethe09) for his contribution to [rules_go](https://github.com/bazelbuild/rules_go) and open-sourcing [buildifier](https://github.com/bazelbuild/buildifier). -* [Peter Mounce](https://github.com/petemounce) for the [chocolatey package of Bazel](https://chocolatey.org/packages/bazel). - - -Thank you all, keep the discussion and bug reports coming! diff --git a/site/blog/_posts/2016-11-04-bazel-build.md b/site/blog/_posts/2016-11-04-bazel-build.md deleted file mode 100644 index 08b2c22d9b..0000000000 --- a/site/blog/_posts/2016-11-04-bazel-build.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -layout: posts -title: We are now bazel.build! ---- - -As you might have seen either in our -[0.4 announcement](/blog/2016/11/02/0.4.0-release.html) or simply by going to -our website, we have recently switched over to the -[bazel.build](https://bazel.build) domain name. - -We decided to switch over to the new .build top-level domain, which reflects -what Bazel is for: building! - -Our old domain, [bazel.io](https://bazel.io), will redirect to -[bazel.build](https://bazel.build) for the forseenable future. diff --git a/site/blog/_posts/2017-02-22-repository-invalidation.md b/site/blog/_posts/2017-02-22-repository-invalidation.md deleted file mode 100644 index 8d25dd53da..0000000000 --- a/site/blog/_posts/2017-02-22-repository-invalidation.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -layout: posts -title: Invalidation of repository rules ---- - -[Remote repositories](/docs/external.html) are the way to use dependencies from -"outside" of the Bazel world in Bazel. Using them, you can download binaries from the -internet or use some from your own host. You can even use -[Skylark](/skylark/repository_rules.html) to define your own repository rules to depend -on a custom package manager or to implement -[auto-configuration rules](/blog/2016/03/31/autoconfiguration.html). - -This post explains when Skylark repositories are invalidated and hence when they are executed. - -## Dependencies - -The implementation attribute of the -[`repository_rule`](https://bazel.build/versions/master/docs/skylark/lib/globals.html#repository_rule) -defines a function (the _fetch_ operation) that is executed inside a -[Skyframe function](/designs/skyframe.html). This function is executed when -one of its dependencies change. - -For repository that are declared `local` (set `local = True` in the call to the -`repository_rule` function), the _fetch_ operation is performed on every call of the -Skyframe function. - -Since a lot of dependencies can trigger this execution (if any part of the `WORKSPACE` -file change for instance), a supplemental mechanism ensure that we re-execute the -_fetch_ operation only when stricly needed for non-`local` repository rules (see the -[design doc](/designs/2016/10/18/repository-invalidation.html) for more details). - -After [cr.bazel.build/8218](https://cr.bazel.build/8218) is released, Bazel will -re-perform the `fetch` operation if and only if any of the following -dependencies change: - -- Skylark files needed to define the repository rule. -- Declaration of the repository rule in the `WORKSPACE` file. -- Value of any environment variable declared with the `environ` attribute of the [`repository_rule`](https://bazel.build/versions/master/docs/skylark/lib/globals.html#repository_rule) function. The value of those environment variable can be enforced from the command line with the -[`--action_env`](/docs/command-line-reference.html#flag--action_env) flag (but this -flag will invalidate every action of the build). -- Content of any file used and referred using a label (e.g., `//mypkg:label.txt` not `mypkg/label.txt`). - -## Good practices regarding refetching - -### Declare your repository as local very carefully - -First and foremost, declaring a repository `local` should be done only for rule that -needs to be eagerly invalidated and are fast to update. For native rule, this is used only -for [`local_repository`](/docs/be/workspace.html#local_repository) and -[`new_local_repository`](/docs/be/workspace.html#new_local_repository). - -### Put all slow operation at the end, resolve dependencies first - -Since a dependency might be unresolved when asked for, the function will be executed -up to where the dependency is requested and all that part will be replayed if the -dependency is not resolved. Put those file dependencies at the top, for instance prefer - -```python -def _impl(repository_ctx): - repository_ctx.file("BUILD", repository_ctx.attr.build_file) - repository_ctx.download("BIGFILE", sha256 = "...") - -myrepo = repository_rule(_impl, attrs = {"build_file": attr.label()}) -``` - -over - -```python -def _impl(repository_ctx): - repository_ctx.download("BIGFILE") - repository_ctx.file("BUILD", repository_ctx.attr.build_file) - -myrepo = repository_rule(_impl, attrs = {"build_file": attr.label()}) -``` - -(in the later example, the download operation will be re-executed if `build_file` is not -resolved when executing the `fetch` operation). - -### Declare your environment variables - -To avoid spurious refetch of repository rules (and the impossibility of tracking all -usages of environmnent variables), only environment variables that have been declared -through the `environ` attribute of the `repository_rule` function are invalidating -the repositories. - -Therefore, if you think you should re-run if an environment variable changes (like -for auto-configuration rules), you should declare those dependencies, or your user -will have to do `bazel clean --expunge` each time they change their environment. diff --git a/site/blog/_posts/2017-02-27-protocol-buffers.md b/site/blog/_posts/2017-02-27-protocol-buffers.md deleted file mode 100644 index 66268743cf..0000000000 --- a/site/blog/_posts/2017-02-27-protocol-buffers.md +++ /dev/null @@ -1,223 +0,0 @@ ---- -layout: posts -title: Protocol Buffers in Bazel ---- - -Bazel currently provides built-in rules for Java, JavaLite and C++. - -[`proto_library`](https://bazel.build/versions/master/docs/be/protocol-buffer.html#proto_library) -is a language-agnostic rule that describes relations between `.proto` files. - -[`java_proto_library`](https://bazel.build/versions/master/docs/be/java.html#java_proto_library), -[`java_lite_proto_library`](https://bazel.build/versions/master/docs/be/java.html#java_lite_proto_library) -and -[`cc_proto_library`](https://bazel.build/versions/master/docs/be/c-cpp.html#cc_proto_library) -are rules that "attach" to `proto_library` and generate language-specific -bindings. - -By making a `java_library` (resp. `cc_library`) depend on `java_proto_library` -(resp. `cc_proto_library`) your code gains access to the generated code. - -## TL;DR - Usage example - -> TIP: -> [https://github.com/cgrushko/proto_library](https://github.com/cgrushko/proto_library) -> contains a buildable example. -> -> NOTE: Bazel 0.4.4 lacks some features the example uses - you'll need to build -> Bazel from head. The easiest is to install Bazel, download Bazel's source -> code, build it (`bazel build //src:bazel`) and copy it somewhere (e.g., `cp -> bazel-bin/src/bazel ~/bazel`) - -### WORKSPACE file - -Bazel's proto rules implicitly depend on the -[https://github.com/google/protobuf](https://github.com/google/protobuf) -distribution (described below, in "Implicit Dependencies and Proto Toolchains"). -The following satisfies these dependencies: - -> TIP: This is a shortened version of -> [https://github.com/cgrushko/proto_library/blob/master/WORKSPACE](https://github.com/cgrushko/proto_library/blob/master/WORKSPACE) - -```python -# proto_library rules implicitly depend on @com_google_protobuf//:protoc, -# which is the proto-compiler. -# This statement defines the @com_google_protobuf repo. -http_archive( - name = "com_google_protobuf", - urls = ["https://github.com/google/protobuf/archive/b4b0e304be5a68de3d0ee1af9b286f958750f5e4.zip"], -) - -# cc_proto_library rules implicitly depend on @com_google_protobuf_cc//:cc_toolchain, -# which is the C++ proto runtime (base classes and common utilities). -http_archive( - name = "com_google_protobuf_cc", - urls = ["https://github.com/google/protobuf/archive/b4b0e304be5a68de3d0ee1af9b286f958750f5e4.zip"], -) - -# java_proto_library rules implicitly depend on @com_google_protobuf_java//:java_toolchain, -# which is the Java proto runtime (base classes and common utilities). -http_archive( - name = "com_google_protobuf_java", - urls = ["https://github.com/google/protobuf/archive/b4b0e304be5a68de3d0ee1af9b286f958750f5e4.zip"], -) -``` - -### BUILD files - -> TIP: This is a shortened version of -> [https://github.com/cgrushko/proto_library/blob/master/src/BUILD](https://github.com/cgrushko/proto_library/blob/master/src/BUILD) - -```python -java_proto_library( - name = "person_java_proto", - deps = [":person_proto"], -) - -cc_proto_library( - name = "person_cc_proto", - deps = [":person_proto"], -) -proto_library( - name = "person_proto", - srcs = ["person.proto"], - deps = [":address_proto"], -) - -proto_library( - name = "address_proto", - srcs = ["address.proto"], - deps = [":zip_code_proto"], -) - -proto_library( - name = "zip_code_proto", - srcs = ["zip_code.proto"], -) -``` - -This file yields the following dependency graph: - - - -Notice how the `proto_library` provide structure for both Java and C++ code -generators, and how there's only one `java_proto_library` even though there -multiple `.proto` files. - -## Benefits - -... in comparison with a macro that's responsible for compiling all `.proto` -files in a project. - -1. Caching + incrementality: changing a single `.proto` only causes the - rebuilding of dependant `.proto` files. This includes not only regenerating - code, but also recompiling it. For large proto graphs this could be - significant. -2. Depend on pieces of a proto graph from multiple places: in the example - above, one can add a `cc_proto_library` that `deps` on `zip_code_proto`, and - including it together with `//src:person_cc_proto` in the same project. - Though they both transitively depend on `zip_code_proto`, there won't be a - linking error. - -## Recommended Code Organization - -1. One proto_library rule per `.proto` file. -2. A file named `foo.proto` will be in a rule named `foo_proto`, which is - located in the same package. -3. A `X_proto_library` that wraps a `proto_library` named `foo_proto` should be - called `foo_X_proto`, and be located in the same package. - -## FAQ - -**Q:** I already have rules named `java_proto_library` and `cc_proto_library`. -Will there be a problem?<br /> -**A:** No. Since Skylark extensions imported through `load` statements take -precedence over native rules with the same name, the new rule should not affect -existing usage of the `java_proto_library` macro. - -**Q:** How do I use gRPC with these rules?<br /> -**A:** The Bazel rules do not generate RPC code since `protobuf` is independent -of any RPC system. We will work with the gRPC team to create Skylark extensions -to do so. ([C++ Issue](https://github.com/grpc/grpc/issues/9873), [Java -Issue](https://github.com/grpc/grpc-java/issues/2756)) - -**Q:** Do you plan to release additional languages?<br /> -**A:** We can relatively easily create `py_proto_library`. Our end goal is to -improve Skylark to the point where these rules can be written in Skylark, making -them independent of Bazel. - -**Q:** How does one use well-known types? (e.g., `any.proto`, -`descriptor.proto`)<br /> -**A:** Once [https://github.com/google/protobuf/issues/2763](https://github.com/google/protobuf/issues/2763) is resolved, the -following should be added to a `.proto` file: `import google/protobuf/any.proto` -and the following: `@com_google_protobuf//:well_known_types_protos` to one's -`proto_library` rule. - -**Q:** Any tips for writing my own such rules?<br /> -**A:** First, make sure you're able to register actions that compile your target -language. (as far as I know, Bazel Python actions are not exposed to Skylark, -for example).<br /> -Second, take extra care to generate unique symbol names and unique filenames. -There's an implicit assumption that different proto rules with different -options, generate different symbols. For example, if you write a new rule -`foo_java_proto_library`, it must not generate symbols that `java_proto_library` -might. The risk is that a binary will contain both, leading to a one-definition -rule violation (e.g., linking errors). The downside is that the binary might -bloat, as it must contain multiple generated code for the same proto. We're -working on a Skylark version of `java_lite_proto_library` which should provide a -good example. - -## Implementation Details - -### Implicit Dependencies and Proto Toolchains - -The `proto_library` rule implicitly depends on `@com_google_protobuf//:protoc`, -which is the protocol buffer compiler. It must be a binary rule (in protobuf, -it's a `cc_binary`). The rule can be overridden using the `--proto_compiler` -command-line flag. - -`X_proto_library` rules implicitly depend on -`@com_google_protobuf_X//:X_toolchain`, which is a `proto_lang_toolchain` rule. -These rules can be overridden using the `--proto_toolchain_for_X` command-line -flags. - -A `proto_lang_toolchain` rule describes how to call the protocol compiler, and -what is the library (if any) that the resulting generated code needs to compile -against. See an [example in the protobuf -repository](https://github.com/google/protobuf/blob/b4b0e304be5a68de3d0ee1af9b286f958750f5e4/BUILD#L773). - -### Bazel Aspects - -The `X_proto_library` rules are implemented using [Bazel -Aspects](https://bazel.build/versions/master/docs/skylark/aspects.html) to have -the best of two worlds - - -1. Only need a single `X_proto_library` rule for an arbitrarily-large proto - graph. -2. Incrementality, caching and no linking errors. - -Conceptually, an `X_proto_library` rule creates a shadow graph of the -`proto_library` it depends on, and each shadow node calls protocol-compiler and -then compiles the generated code. This way, if there are multiple paths from a -rule to a `proto_library` through `X_proto_library`, they all share the same -node. - -### Descriptor Sets - -When compiled on the command-line, a `proto_library` creates a descriptor set -for the messages it `srcs`. The file is a serialized `FileDescriptorSet`, which -is described in -[https://developers.google.com/protocol-buffers/docs/techniques#self-description](https://developers.google.com/protocol-buffers/docs/techniques#self-description). - -One use case for the descriptor set is generating code without having to parse -`.proto` files. -([https://github.com/google/protobuf/issues/2725](https://github.com/google/protobuf/issues/2725) -tracks this ability in the protobuf compiler) - -The aforementioned file only contains information about the `.proto` files -directly mentioned by a `proto_library` rule; the collection of transitive -descriptor sets is available through the 'proto.transitive_descriptor_sets' -Skylark provider. See [documentation in -ProtoSourcesProvider](https://github.com/bazelbuild/bazel/blob/5dbb23ba44ec0037cf0944b17716ea3f08a69c27/src/main/java/com/google/devtools/build/lib/rules/proto/ProtoSourcesProvider.java#L121). - -_By [Carmi Grushko](https://github.com/cgrushko)_ diff --git a/site/blog/_posts/2017-02-28-google-summer-of-code.md b/site/blog/_posts/2017-02-28-google-summer-of-code.md deleted file mode 100644 index 396df77fe2..0000000000 --- a/site/blog/_posts/2017-02-28-google-summer-of-code.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -layout: posts -title: A Google Summer of Code with Bazel ---- - -I'm happy to announce that Bazel has been accepted as -[a mentor organization](https://summerofcode.withgoogle.com/organizations/6293241017139200/) -for the -[Google Summer of Code 2017](https://developers.google.com/open-source/gsoc/). -If you are a student and interested in working on Bazel this summer, please read -on. - -Take a look at our -[ideas page](https://github.com/bazelbuild/bazel/wiki/Project-Ideas): it is -not exhaustive and we may extend it over time, but it should give you a rough -idea of what you could work on. Feel free to come up with your new ideas or -suggest variations on our proposals. Not all projects on the page will be taken: -we expect to accept up to three students. Students will not work with a single -mentor, you can expect to interact with multiple people from the Bazel team -(although there will be a main contact point). This will ensure you'll get -timely responses and assistance, even if one of us goes on vacation. - -This is the first time we participate in Google Summer of Code, please bear with -us if you miss some information. We will update our ideas page to answer the -most frequent questions. - -If you have any question, please contact us on bazel-core@googlegroups.com - -I'm looking forward to hearing from you, - -_Laurent Le Brun, on behalf of the Bazel mentors._ diff --git a/site/blog/_posts/2017-03-07-java-sandwich.md b/site/blog/_posts/2017-03-07-java-sandwich.md deleted file mode 100644 index e07931597e..0000000000 --- a/site/blog/_posts/2017-03-07-java-sandwich.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -layout: posts -title: Skylark and Java rules interoperability ---- - -As of Bazel 0.4.4, Java compilation is possible from a Skylark rule. This -facilitates the Skylark and Java interoperability and allows creating what we -call _Java sandwiches_ in Bazel. - -## What is a Bazel Java sandwich? - -A Java sandwich refers to custom rules written in Skylark being able to depend -on Bazel native rules (e.g. `java_library`) and the other way around. A typical -Java sandwich in Bazel could be illustrated like this: - -```python -java_library(name = "top", ...) -java_skylark_library(name = "middle", deps = [":top", ...], ...) -java_library(name = "bottom", deps = [":middle", ...], ...) -``` -## Built-in support for Java - -In Skylark, an interface to built-in Java functionality is available via the `java_common` module. -The full API can be found in [the documentation](https://bazel.build/versions/master/docs/skylark/lib/java_common.html). - -### `java_common.compile` - -Compiles Java source files/jars from the implementation of a Skylark rule and -returns a `java_common.provider` that encapsulates the compilation details. - -### `java_common.merge` - -Merges the given providers into a single `java_common.provider`. - - -## Examples - -To allow other Java rules (native or custom) to depend on a Skylark rule, the -Skylark rule should return a `java_common.provider`. All native Java rules -return `java_common.provider` by default, which makes it possible for any Java -related Skylark rule to depend on them. - -For now, there are 3 ways of creating a `java_common.provider`: - -1. The result of `java_common.compile`. -2. Fetching it from a Java dependency. -3. Merging multiple `java_common.provider` instances using `java_common.merge`. - -### Using the Java sandwich with compilation example - -This example illustrates the typical Java sandwich described above, that will -make use of Java compilation: - -```python -java_library(name = "top", ...) -java_skylark_library(name = "middle", deps = [":top", ...], ...) -java_library(name = "bottom", deps = [":middle", ...], ...) -``` - -In the BUILD file we load the Skylark rule and have the rules: - -```python -load(':java_skylark_library.bzl', 'java_skylark_library') - -java_library( - name = "top", - srcs = ["A.java"], - deps = [":middle"] -) - -java_skylark_library( - name = "middle", - srcs = ["B.java"], - deps = [":bottom"] -) - -java_library( - name = "bottom", - srcs = ["C.java"] -) -``` - -The implementation of `java_skylark_library` rule does the following: - -1. Collects all the `java_common.provider`s from its dependencies and merges -them using `java_common.merge`. -2. Creates an artifact that will be the output jar of the Java compilation. -3. Compiles the specified Java source files using `java_common.compile`, passing -as dependencies the collected `java_common.provider`s. -4. Returns the output jar and the `java_common.provider` resulting from the -compilation. - -```python -def _impl(ctx): - deps = [] - for dep in ctx.attr.deps: - if java_common.provider in dep: - deps.append(dep[java_common.provider]) - - output_jar = ctx.new_file("lib" + ctx.label.name + ".jar") - - compilation_provider = java_common.compile( - ctx, - source_files = ctx.files.srcs, - output = output_jar, - javac_opts = [], - deps = deps, - strict_deps = "ERROR", - java_toolchain = ctx.attr._java_toolchain, - host_javabase = ctx.attr._host_javabase - ) - return struct( - files = set([output_jar]), - providers = [compilation_provider] - ) - -java_skylark_library = rule( - implementation = _impl, - attrs = { - "srcs": attr.label_list(allow_files=True), - "deps": attr.label_list(), - "_java_toolchain": attr.label(default = Label("@bazel_tools//tools/jdk:toolchain")), - "_host_javabase": attr.label(default = Label("//tools/defaults:jdk")) - }, - fragments = ["java"] -) -``` - -### Just passing around information about Java rules example - -In some use cases there is no need for Java compilation, but rather just passing -information about Java rules around. A Skylark rule can have some other -(irrelevant here) purpose, but if it is placed somewhere between two Java rules -it should not lose information from bottom to top. - -In this example we have the same Bazel sandwich as above: - -```python -java_library(name = "top", ...) -java_skylark_library(name = "middle", deps = [":top", ...], ...) -java_library(name = "bottom", deps = [":middle", ...], ...) -``` - -only that `java_skylark_library` won't make use of Java compilation, but will -make sure that all the Java information encapsulated by the Java library -`bottom` will be passed on to the Java library `top`. - -The BUILD file is identical to the one from the previous example. - -The implementation of `java_skylark_library` rule does the following: - -1. Collects all the `java_common.provider`s from its dependencies -2. Returns the `java_common.provider` that resulted from merging the collected -dependencies. - -```python -def _impl(ctx): - deps = [] - for dep in ctx.attr.deps: - if java_common.provider in dep: - deps.append(dep[java_common.provider]) - deps_provider = java_common.merge(deps) - return struct( - providers = [deps_provider] - ) - -java_skylark_library = rule( - implementation = _impl, - attrs = { - "srcs": attr.label_list(allow_files=True), - "deps": attr.label_list(), - "_java_toolchain": attr.label(default = Label("@bazel_tools//tools/jdk:toolchain")), - "_host_javabase": attr.label(default = Label("//tools/defaults:jdk")) - }, - fragments = ["java"] -) -``` -## More to come - -Right now there is no way of creating a `java_common.provider` that encapsulates -compiled code (and its transitive dependencies), other than -`java_common.compile`. For example one may want to create a provider from a -`.jar` file produced by some other means. - -Soon there will be support for use cases like this. Stay tuned! - -If you are interested in tracking the progress on Bazel Java sandwich you can -subscribe to [this Github issue](https://github.com/bazelbuild/bazel/issues/2614). - -_[Irina Iancu](https://github.com/iirina), on behalf of the Bazel Java team_ diff --git a/site/blog/_posts/2017-03-21-design-of-skylark.md b/site/blog/_posts/2017-03-21-design-of-skylark.md deleted file mode 100644 index 849da98cf9..0000000000 --- a/site/blog/_posts/2017-03-21-design-of-skylark.md +++ /dev/null @@ -1,96 +0,0 @@ ---- -layout: posts -title: A glimpse of the design of Skylark ---- - -This blog post describes the design of Skylark, the language used to specify -builds in Bazel. - -## A brief history - -Many years ago, code at Google was built using Makefiles. As [other people -noticed](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/03/hadrian.pdf), -Makefiles don't scale well with a large code base. A temporary solution was to -generate Makefiles using Python scripts, where the description of the build was -stored in `BUILD` files containing calls to the Python functions. But this -solution was way too slow, and the bottleneck was Make. - -The project Blaze (later open-sourced as Bazel) was started in 2006. It used a -simple parser to read the `BUILD` files (supporting only function calls, list -comprehensions and variable assignments). When Blaze could not directly parse a -`BUILD` file, it used a preprocessing step that ran the Python interpreter on -the user `BUILD` file to generate a simplified `BUILD` file. The output was used -by Blaze. - -This approach was simple and allowed developers to create their own macros. But -again, this led to lots of problems in terms of maintenance, performance, and -safety. It also made any kind of tooling more complicated, as Blaze was not able -to parse the `BUILD` files itself. - -In the current iteration of Bazel, we've made the system saner by removing the -Python preprocessing step. We kept the Python syntax, though, in order to -migrate our codebase. This seems to be a good idea anyway: Many people like the -syntax of our `BUILD` files and other build tools (e.g. -[Buck](https://buckbuild.com/concept/build_file.html), -[Pants](http://www.pantsbuild.org/build_files.html), and -[Please](https://please.build/language.html)) have adopted it. - -## Design requirements - -We decided to separate description of the build from the extensions (macros and -rules). The description of the build resides in `BUILD` files and the extensions -reside in `.bzl` files, although they are all evaluated with the same -interpreter. We want the code to be easy to read and maintain. We designed Bazel -to be used by thousands of engineers. Most of them are not familiar with build -systems internals and most of them don't want to spend time learning a new -language. `BUILD` files need to be simple and declarative, so that we can build -tools to manipulate them. - -The language also needed to: - -* Run on the JVM. Bazel is written in Java. The data structures should be - shared between Bazel and the language (due to memory requirements in large - builds). - -* Use a Python syntax, to preserve our codebase. - -* Be deterministic and hermetic. We have to guarantee that the execution of - the code will always yield the same results. For example, we forbid access - to I/O and date and time, and ensure deterministic iteration order of - dictionaries. - -* Be thread-safe. We need to evaluate a lot of `BUILD` files in parallel. - Execution of the code needs to be thread-safe in order to guarantee - determinism. - -Finally, we have performance concerns. A typical `BUILD` file is simple and can -be executed quickly. In most cases, evaluating the code directly is faster than -compiling it first. - -## Parallelism and imports - -One special feature of Skylark is how it handles parallelism. In Bazel, a large -build require the evaluation of hundreds of `BUILD` files, so we have to load -them in parallel. Each `BUILD` file may use any number of extensions, and those -extensions might need other files as well. This means that we end up with a -graph of dependencies. - -Bazel first evaluates the leaves of this graph (i.e. the files that have no -dependencies) in parallel. It will load the other files as soon as their -dependencies have been loaded, which means the evaluation of `BUILD` and `.bzl` -files is interleaved. This also means that the order of the `load` statements -doesn't matter at all. - -Each file is loaded at most once. Once it has been evaluated, its definitions -(the global variables and functions) are cached. Any other file can access the -symbols through the cache. - -Since multiple threads can access a variable at the same time, we need a -restriction on side-effects to guarantee thread-safety. The solution is simple: -when we cache the definitions of a file, we "freeze" them. We make them -read-only, i.e. you can iterate on an array, but not modify its elements. You -may create a copy and modify it, though. - -In a future blog post, we'll take a look at the other features of the language. - -_By [Laurent Le Brun](https://github.com/laurentlb)_ diff --git a/site/blog/_posts/2017-04-21-JDK7-deprecation.md b/site/blog/_posts/2017-04-21-JDK7-deprecation.md deleted file mode 100644 index f010dbaa58..0000000000 --- a/site/blog/_posts/2017-04-21-JDK7-deprecation.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -layout: posts -title: JDK7 deprecation ---- - -The Bazel team has been maintaining a separate, stripped-down build of Bazel -that runs with JDK 7. The 0.5.1 release will no longer provide this special -version. - -To address the problem of JDK 8 not being available on some machines, starting -with version 0.5.0, our installer will embed a JDK by default. - -If you have any concerns, please reach out to -[bazel-discuss@googlegroups.com](mailto:bazel-discuss@googlegroups.com). - -__Recap:__ - -__0.5.0__: - - * `bazel-0.5.0-installer.sh`: default version, with embedded JDK. - * `bazel-0.5.0-without-jdk-installer.sh`: version without embedded JDK. - * `bazel-0.5.0-jdk7-installer.sh`: last release compatible with JDK 7. - -__0.5.1__: - - * `bazel-0.5.1-installer.sh`: default version, with embedded JDK. - * `bazel-0.5.1-without-jdk-installer.sh`: version without embedded JDK. - -__Migration path:__ - -If you are currently using the Bazel with JDK 7, then starting with version -0.5.0 you must start using the default installer. - -If you are currently using the default installer and do not want to use a -version with embedded JDK, then use the `-without-jdk` version. - -__Note:__ - -Homebrew and debian packages do not contain the embedded JDK. This change only -affects the shell installers. - -__Thanks:__ - -Thanks everybody for bearing with all the JDK 7 related issues, including the -Java team at Google, in particular -[Liam Miller-Cushon](https://github.com/cushon). - -Special thanks to [Philipp Wollermann](https://github.com/philwo) who made this -new installer possible. - - diff --git a/site/blog/_posts/2017-05-26-Bazel-0-5-0-release.md b/site/blog/_posts/2017-05-26-Bazel-0-5-0-release.md deleted file mode 100644 index 3bd0de396a..0000000000 --- a/site/blog/_posts/2017-05-26-Bazel-0-5-0-release.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -layout: posts -title: Bazel 0.5.0 Released ---- - -We are delighted to announce the [0.5.0 release of -Bazel](https://github.com/bazelbuild/bazel/releases/tag/0.5.0) (follow the link -for the full release notes and list of changes). - -This release simplifies Bazel installation on Windows and platforms where a JDK -is not available. It solidifies the Build Event Protocol and [Remote Execution -APIs](https://docs.google.com/document/d/1AaGk7fOPByEvpAbqeXIyE8HX_A3_axxNnvroblTZ_6s/edit). - -> Note: Bazel release 0.5.0 contains a bug in the compiler detection on macOS which -> requires Xcode and the iOS tooling to be installed -> ([corresponding issue #3063](https://github.com/bazelbuild/bazel/issues/3063)). -> If you had Command Line Tools installed, you also need to switch to Xcode using -> `sudo xcode-select -s /Applications/Xcode.app/Contents/Developer`. - -## Improvements from our roadmap - -### Bundled JDK - -As announced earlier, when using an install script, bazel now comes by default -bundled with JDK 8. This means fewer steps required to install Bazel. Read more -about JDK 7 deprecation in [the related blog -post](https://bazel.build/blog/2017/04/21/JDK7-deprecation.html). - -### Windows support: now in beta - -Bazel on Windows is now easier to install: it is no longer linked with MSYS. A -following blog post will detail this further. Bazel is now able to build Java, -C++ and Python on Windows. - -### Build Event Protocol - -The [Build Event -Protocol](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto) -is now available as an experimental option; it enables programmatic subscription -to Bazel's events (build started, action status, target completed, test -results…). Currently, the protocol can only be written to a file. A gRPC -transport is already in the works and will be added in the next minor release. -The API will be stabilized in 0.5.1. - -### Coverage support for pure Java targets - -Use bazel coverage //my:target to generate coverage information from a -java\_test. - -## Other major changes since 0.4.0 - -### New rules - -New rules in Bazel: -[proto\_library](https://bazel.build/versions/master/docs/be/protocol-buffer.html#proto_library), -[java\_lite\_proto\_library](https://bazel.build/versions/master/docs/be/java.html#java_lite_proto_library), -[java\_proto\_library](https://bazel.build/versions/master/docs/be/java.html#java_proto_library) -and -[cc\_proto\_library](https://bazel.build/versions/master/docs/be/c-cpp.html#cc_proto_library). - -### New Apple rules - -There is a new repository for building for Apple platforms: -[https://github.com/bazelbuild/rules\_apple](https://github.com/bazelbuild/rules_apple). -These rules replace the deprecated iOS/watchOS rules built into Bazel. By -rebuilding the rules from the ground up in Skylark and hosting them separately, -we can more quickly fix bugs and implement new Apple features and platform -versions as they become available. - -### Android Support Improvements - -- Integration with the Android Support Repository libraries in - android\_sdk\_repository. -- Support for Java 8 in Android builds with --experimental\_desugar\_for\_android. - See [Android Studio's - documentation](https://developer.android.com/studio/preview/features/java8-support.html) - for more details about Android's Java 8 language features. -- Multidex is now fully supported via - [android\_binary.multidex](https://bazel.build/versions/master/docs/be/android.html#android_binary.multidex). -- android\_ndk\_repository now supports Android NDK 13 and NDK 14. -- APKs are now signed with both APK Signature V1 and V2. - See [Android - documentation](https://source.android.com/security/apksigning/v2.html) for more - details about APK Signature Scheme v2. - -### Remote Execution API - -We fixed a number of bugs in the Remote Execution implementation. The final RPC -API design has been sent to -[bazel-discuss@](https://groups.google.com/forum/#!forum/bazel-discuss) for -discussion (see [Design Document: Remote Execution -API](https://docs.google.com/document/d/1AaGk7fOPByEvpAbqeXIyE8HX_A3_axxNnvroblTZ_6s/edit#heading=h.ole76l21af90)) -and it should be finalized in the 0.6.0 release. The final API should only be a -minor change compared to the implementation in this 0.5.0 release. - -## Skylark - -- Declared Providers are now implemented and - [documented](https://bazel.build/versions/master/docs/skylark/rules.html#providers). - They enable more robust and clearly defined interfaces between different - rules and aspects. We recommend using them for all rules and aspects. -- The type formerly known as 'set' is now called 'depset'. Depsets make your - rules perform much better, allowing rules memory consumption to scale - linearly instead of quadratically with build graph size - make sure you have - read the - [documentation on depsets](https://bazel.build/versions/master/docs/skylark/depsets.html). - -## Finally... - -A big thank you to our community for your continued support. -Particular shout-outs to Peter Mounce for the [Chocolatey Windows -package](https://bazel.build/versions/master/docs/install-windows.html) and Yuki -Yugui Sonoda for maintaining [rules\_go](https://github.com/bazelbuild/rules_go) -(they both received an [open source peer -bonus](https://opensource.googleblog.com/2017/03/the-latest-round-of-google-open-source.html) -from Google). - -Thank you all, keep the -[questions](http://stackoverflow.com/questions/tagged/bazel) and [bug -reports](https://github.com/bazelbuild/bazel/issues) coming! - -See the full list of changes on [GitHub](https://github.com/bazelbuild/bazel/releases/tag/0.5.0). diff --git a/site/blog/feed.xml b/site/blog/feed.xml deleted file mode 100644 index 2d039368ce..0000000000 --- a/site/blog/feed.xml +++ /dev/null @@ -1,20 +0,0 @@ ---- ---- -<?xml version="1.0" encoding="UTF-8" ?> -<rss version="2.0"> - -<channel> - <title>Bazel Blog</title> - <link>http://bazel.build</link> - <description>Fast, correct builds.</description> - {% for post in site.categories.blog limit:10 %} - <item> - <title>{{ post.title }}</title> - <link>{{ post.url }}</link> - <description>{{ post.content | xml_escape }}</description> - <pubDate>{{ post.date | date: "%a, %d %b %Y %H:%M:%S %z" }}</pubDate> - </item> - {% endfor %} -</channel> - -</rss> diff --git a/site/blog/index.md b/site/blog/index.md deleted file mode 100644 index 5f2d3761ba..0000000000 --- a/site/blog/index.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -layout: blog -title: Blog ---- - -{% for post in site.categories.blog %} - -<div class="blog-post"> - <h1 class="blog-post-title"><a href="{{ post.url }}">{{ post.title }}</a></h1> - <div class="blog-post-meta"> - <span class="text-muted">{{ post.date | date_to_long_string }}</span> - </div> - {{ post.content }} -</div> - -{% endfor %} diff --git a/site/contributing.md b/site/contributing.md deleted file mode 100644 index af77efd8f7..0000000000 --- a/site/contributing.md +++ /dev/null @@ -1,180 +0,0 @@ ---- -layout: contribute -title: Contributing to Bazel ---- - -# Contributing to Bazel - -<p class="lead">We welcome contributions! This page covers setting up your -machine to develop Bazel and, when you've made a patch, how to submit it.</p> - -## How can I contribute to Bazel? - -In general, we prefer contributions that fix bugs or add features (as opposed to -stylistic, refactoring, or "cleanup" changes). - -Please check with us on the -[dev list](https://groups.google.com/forum/#!forum/bazel-dev) before investing -a lot of time in a patch. Meet other Bazel contributors on [IRC](http://webchat.freenode.net) -(irc.freenode.net#bazel). - -### Patch Acceptance Process - - -1. Read the [Bazel governance plan](governance.html). -2. Discuss your plan and design, and get agreement on our [mailing list](https://groups.google.com/forum/#!forum/bazel-dev). -3. Prepare a git commit that implements the feature. Don't forget to add tests. -4. Ensure you've signed a [Contributor License Agreement](https://cla.developers.google.com). -5. Create a new code review. You can use a GitHub pull request, or a code - review on [Gerrit](https://bazel-review.googlesource.com). - * To use GitHub, send a pull request. If you're new to GitHub, read [about - pull requests](https://help.github.com/articles/about-pull-requests/). - * To use Gerrit, you must: - * Have an automatically generated "Change Id" line in your commit - message. If you haven't used Gerrit before, it will print a bash - command to create the git hook and then you will need to run - `git commit --amend` to add the line. - * To create a code review on Gerrit, run: - - ``` - git push https://bazel.googlesource.com/bazel HEAD:refs/for/master - ``` - - The HTTP password required by Gerrit can be obtained from your - [Gerrit settings page](https://bazel-review.googlesource.com/#/settings/http-password). - See the [Gerrit documentation](https://gerrit-review.googlesource.com/Documentation/user-upload.html) - for more information about uploading changes. -6. Wait for a Bazel team member to assign you a reviewer. - It should be done in 2 business days (excluding holidays in the USA and - Germany). If you do not get a reviewer within that time frame, you can ask - for one by sending a mail to [bazel-sheriff@googlegroups.com](mailto:bazel-sheriff@googlegroups.com). - You can also assign yourself a reviewer if you know who the reviewer should - be (e.g., because they reviewed an earlier related change). -7. Complete a code review. Amend your existing commit and re-push to make - changes to your patch. -8. An engineer at Google applies the patch to our internal version control - system. The patch is exported as a Git commit, at which point the GitHub - or Gerrit code review is closed. - -## Setting up your coding environment - -For now we have support for IntelliJ, and partial support for the Eclipse IDE -for Java. We don't have IDE support for other languages in Bazel right now. - -### Preparations - -* [Install Bazel](https://bazel.build/versions/master/docs/install.html) on your - system. Note that for developing Bazel, you need the latest released version - of Bazel. -* Clone Bazel's Git repository from Gerrit: - * `git clone https://bazel.googlesource.com/bazel` -* Try to build Bazel: - * `cd bazel && bazel build //src:bazel` -* This should produce a working Bazel binary in `bazel-bin/src/bazel`. - -If everything works fine, feel free to configure your favorite IDE in the -following steps. - -### Creating an IntelliJ project - -To work with IntelliJ: - -* Install Bazel's [IntelliJ plug-in](https://ij.bazel.build). -* Set the path to the Bazel binary in the plugin preferences - (`Preferences` > `Other Settings` > `Bazel Settings`). -* Import the Bazel workspace as a Bazel project - (`File` > `Import Bazel Project...`) with the following settings: - * Use existing bazel workspace: choose your cloned Git repository. - * Select `Import from workspace` and choose the `scripts/ij.bazelbuild` - file as the `Project view`. -* Download [Google's Java Code Style Scheme file for IntelliJ](https://github.com/google/styleguide/blob/gh-pages/intellij-java-google-style.xml), - import it (go to `Preferences` > `Editor` > `Code Style` > `Java`, click `Manage`, then `Import`) - and use it when working on Bazel's code. - -### Creating an Eclipse project - -To work with Eclipse: - -* [Install Bazel](https://bazel.build/versions/master/docs/install.html) on your system. -* Clone Bazel's Git repository from Gerrit: - * `git clone https://bazel.googlesource.com/bazel` -* Install the [e4b](https://github.com/bazelbuild/e4b) plugin. -* Change the path to the Bazel binary in the plugin preferences. -* Import the Bazel workspace as a Bazel project (`File` > `New` > `Other` > - `Import Bazel Workspace`). -* Select `src > main > java` and `src > test > java` as directories and add - `//src/main/java/...` and `//src/test/java/...` as targets. -* Download [Google's Java Code Style Scheme file for Eclipse](https://github.com/google/styleguide/blob/gh-pages/eclipse-java-google-style.xml) and use it when working on Bazel's code. - -<a name="compile-bazel"></a> -### Compiling Bazel - -To test out bazel, you need to compile it. To compile a development version of -Bazel, you need a the latest released version of bazel, which can be -[compiled from source](/versions/master/docs/install.html#compiling-from-source). - -`bazel build //src:bazel` builds the Bazel binary using `bazel` from your PATH -and the resulting binary can be found at `bazel-bin/src/bazel`. This is the -recommended way of rebuilding Bazel once you have bootstrapped it. - -In addition to the Bazel binary, you might want to build the various tools Bazel -uses. They are located in `//src/java_tools/...`, `//src/objc_tools/...` and -`//src/tools/...` and their directories contain README files describing their -respective utility. - -When modifying Bazel, you want to make sure that the following still works: - -* Build a distribution archive with `bazel build //:bazel-distfile`. After - unzipping it in a new empty directory, run `bash compile.sh all` there. - It rebuilds Bazel with `./compile.sh`, Bazel with the - `compile.sh` Bazel and Bazel with the Bazel-built binary. It compares if the - constructed Bazel builts are identical and then runs all bazel tests with - `bazel test //src/... //third_party/ijar/...`. This is what we use at Google - to ensure that we don't break Bazel when pushing new commits, too. - -### Debugging Bazel - -Start creating a debug configuration for both C++ and Java in your `.bazelrc` -with the following: - -``` -build:debug -c dbg -build:debug --javacopt="-g" -build:debug --copt="-g" -build:debug --strip="never" -``` - -Then you can rebuild Bazel with `bazel build --config debug //src:bazel` and use -your favorite debugger to start debugging. - -For debugging the C++ client you can just run it from gdb or lldb as you normally would. -But if you want to debug the Java code, you must attach to the server using the following: - -* Run Bazel with debugging option `--host_jvm_debug` before the - command (e.g., `bazel --batch --host_jvm_debug build //src:bazel`). -* Attach a debugger to the port 5005. With `jdb` for instance, - run `jdb -attach localhost:5005`. From within Eclipse, use the - [remote Java application launch - configuration](http://help.eclipse.org/luna/index.jsp?topic=%2Forg.eclipse.jdt.doc.user%2Ftasks%2Ftask-remotejava_launch_config.htm). -* Our IntelliJ plugin has built-in - [debugging support](https://ij.bazel.build/docs/run-configurations.html) - -## Bazel's code description - -Bazel is organized in several parts: - -* Client code in `src/main/cpp` provides the command-line interface. -* Protocol buffers in `src/main/protobuf`. -* Server code in `src/main/java` and `src/test/java`. - * Core code which is mostly composed of [SkyFrame](designs/skyframe.html) and some - utilities. - * Rules written in Bazel's extension language - [Skylark](docs/skylark/index.html) are defined in `tools/build_rules`. If - you want to add rules, consider using [Skylark](docs/skylark/index.html) - first. - * Builtin rules in `com.google.devtools.build.lib.rules` and in - `com.google.devtools.build.lib.bazel.rules`. You might want to read about - the [Challenges of Writing Rules](docs/rule-challenges.html) first. -* Java native interfaces in `src/main/native`. -* Various tooling for language support (see the list in the - [compiling Bazel](#compile-bazel) section). diff --git a/site/designs/_posts/2015-03-06-bazel-init.md b/site/designs/_posts/2015-03-06-bazel-init.md deleted file mode 100644 index 1cd6f258ac..0000000000 --- a/site/designs/_posts/2015-03-06-bazel-init.md +++ /dev/null @@ -1,304 +0,0 @@ ---- -layout: contribute -title: bazel init a.k.a ./configure for Bazel ---- - -__This design document has been replaced by -[Skylark Remote Repositories](/designs/2015/07/02/skylark-remote-repositories.html) -and is maintained here just for reference__ - -# Design Document: bazel init a.k.a ./configure for Bazel -_A configuration mechanism for Bazel_ - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status:** deprecated, replaced by [Skylark Remote Repositories](/designs/2015/07/02/skylark-remote-repositories.html) - -**Author:** dmarting@google.com - -**Design document published**: 06 March 2015 - -## I. Rationale - -Bazel tooling needs special setup to work. For example, C++ crosstool -configuration requires path to GCC or Java configuration requires the -path to the JDK. Autodetecting those paths from Bazel would be broken -because each ruleset requires its own configuration (C++ CROSSTOOL -information is totally different from JDK detection or from go root -detection). Therefore, providing a general mechanism to configure -Bazel tooling seems natural. To have Bazel self-contained, we will -ship this mechanism as an additional command of Bazel. Because this -command deals with non-hermetic parts of Bazel, this command should -also group all non-hermetic steps (i.e. it should fetch the -dependencies from the remote repositories) so a user can run it and -get on a plane with everything needed. - -## II. Considered use-cases - -We consider the 3 following use-cases: - - - __UC1.__ The user wants to not worry about tools configuration and - use the default one for golden languages (Java, C++, Shell) and - wants to also activate an optional language (Go). No configuration - information (aka `tools` package) should be checked into the - version control system. - - __UC2.__ The user wants to tweak Java configuration but not C++. Of - course, the user wants his tweak to be shared with his team (i.e. - `tools/jdk` should be checked into the version control system). - However, the user does not want to have C++ information (i.e. - `tools/cpp`) in the VCS. - - __UC3.__ The user wants his build to be hermetic and he wants to - set up everything in his `tools` directory (Google use-case). - -### Notes - -This document addresses the special case of the configuration of the -`tools` package, mechanisms presented here could be extended to any -dependency that needs to be configured (e.g., detecting the installed -libncurse) but that is out of the scope of this document. - -Anywhere in this document we refer to the `tools` package as the -package that will receive the current `tools` package content, it does -not commit to keep that package name. - -## III. Requirements - -### `bazel init` should: - - _a1._ Not be available in hermetic version (i.e. Google version of - Bazel, a.k.a Blaze). - - _a2._ Allow per-language configuration. I.e., Java and C++ tooling - configuration should be separated. - - _a3._ Allow Skylark add-ons to specify their configuration, this - should be pluggable so we can actually activate configuration per - rule set. - - _a4._ Support at least 3 modes corresponding to each envisioned - use-cases: - * __UC1.__: __installed__ (default mode): a "hidden" `tools` - package contains the detected tool paths (`gcc`, the JDKs, ...) - as well as their configuration (basically the content of the - current `//tools` package). This package should be constructed - as much as possibly automatically with a way for the user to - overwrite detected settings. - * __UC2.__: __semi-hermetic__: the "hidden" `tools` package is used - only for linking the actual tool paths but the configuration - would be checked-in into the workspace (in a similar way that - what is currently done in Bazel). The "hidden" `tools` package - could contains several versions of the same tools (e.g., jdk-8, - jdk-7, ...) and the workspace link to a specific one. - * __UC3.__: __hermetic__: this is the Google way of thing: the - user check-in everything that he thinks belong to the workspace - and the init command should do nothing. - - _a5._ Support explicit reconfiguration. If the configuration - mechanism changes or the user wants to tune the configuration, it - should support to modify the configuration, i.e., update the - various paths or change the default options. - -### `bazel init` could: - - - _b1._: Initialize a new workspace: as it would support configuring - a whole tool directory, it might be quite close to actually - initializing a new workspace. - -## IV. User interface - -To be efficient, when the `tools` directory is missing, `bazel build` -should display an informative error message to actually run `bazel -init`. - -Configuration is basically just setting a list of build constants like -the path to the JDK, the list of C++ flags, etc... - -When the user type `bazel init`, the configuration process starts with -the default configuration (e.g., configure for “gold features” such as -C++, Java, sh_, ...). It should try to autodetect as much as possible. -If a language configuration needs something it cannot autodetect, then -it can prompt the user for the missing information and the -configuration can fail if something is really wrong. - -On default installation, `bazel init` should not prompt the user at -all. When the process finishes, the command should output a summary of -the configuration. The configuration is then stored in a "hidden" -directory which is similar to our current `tools` package. By default, -the labels in the configuration would direct to that package (always -mapped as a top-level package). The "hidden:" directory would live in -`$(output_base)/init/tools` and be mapped using the package path -mechanism. The `--overwrite` option would be needed to rerun the -automatic detection and overwrite everything including the eventual -user-set options. - -For the hermetic mode, the user has to recreate the default tools -package inside the workspace. If the user has a package with the same -name in the workspace, then the "hidden" directory should be ignored -(--package_path). - -To set a configuration option, the user would type `bazel init -java:jdk=/path/to/jdk` or to use the autodetection on a specific -option `bazel init java:jdk`. The list of settings group could be -obtained by `bazel init list` and the list of option with their value -for a specific language by `bazel init list group`. `bazel init list -all` could give the full configuration of all activated groups. - -_Prospective idea:_ Bazel init should explore the BUILD file to find the -Skylark `load` statements, determine if there is an associated init -script and use it. - -## V. Developer interface - -This section presents the support for developer that wants to add -autoconfiguration for a ruleset. The developer adding a configuration -would provide with a configuration script for it. This script will be -in charge of creating the package in the tools directory during `bazel -init` (i.e., the script for Java support will construct the -//tools/jdk package in the "hidden" package path). - -Because of skylark rules and the fact that the configuration script -should run before having access to the C++ and Java tooling, this -seems unreasonable to use a compiled language (Java or C++) for this -script. We could use the Skylark support to make it a subset of python -or we could use a bash script. Python support would be portable since -provided by Bazel itself and consistent with skylark. It also gives -immediate support for manipulating BUILD files. So keeping a -"skylark-like" syntax, the interface would look like: - -```python -configuration( - name, # name of the tools package to configure - autodetect_method, # the auto detection method - generate_method, # the actual package generation - load_method, # A method to load the attributes presented - # to the user from the package - attrs = { # List of attributes this script propose - "jdk_path": String, - "__some_other_path": String, # not user-settable - "jdk_version": Integer, - }) -``` - -Given that interface, an initial run of `bazel init` would do: - - 1. Find all language configuration scripts - 2. Run `load_method` for each script - 3. Run `autodetect_method` for each script. Replace non loaded - attribute (attribute still undefined after `load_method`) if and - only if `--rerun` option is provided - 4. Run `generate_method` for each script - 5. Fetch all non up to date dependencies of remote repository - -See Appendix B for examples of such methods. - -## VI. Implementation plan - - 1. Add the hidden tools directory and have it binded with package - path when no tools directory exists. The hidden tools directory - will have a WORKSPACE file and will have an automatic local - repository with the "init" name so that we can actually bind - targets from it into our workspace. - 2. Add `bazel init` that support the configuration for native - packages in Java, that is: Java, C++, genrule and test. This - would create the necessary mechanisms for supporting the - developer and the basic user interface. This commands will be - totally in Java for now and should trigger the fetch part of the - remote repository. - 3. Design and implement the language extension ala Skylark using the - design for the Java version of point 2. - 4. Convert the existing configuration into that language. - 5. Integrate the configuration with Skylark (i.e. Skylark rules - writer can add configuration step). We should here decide on - how it should be included (as a separate script? how do we - ship a skylark rule set? can we have load statement loading - a full set of rules?). - 6. Create configuration for the existing skylark build rules. If - we support load statement with label, we can then create a - repository for Skylark rules. - -## Appendix A. Various comments - - 1. We should get rid of the requirement for a `tools/defaults/BUILD` file. - 2. To works correctly, we need some local caching of the bazel - repository so tools are available. We could have bazelrc specify a - global path to the local cache (with `/etc/bazel.bazelrc` being loaded - first to `~/.bazelrc`). We could use a `~/.bazel` directory to put an - updatable tools cache also. This is needed because user probably want - to initialize a workspace tooling on a plane - 3. This proposal would probably add a new top-level package. We - should really take care of the naming convention for default top - packages (i.e., `tools`, `tools/defaults`, `visibility`, `external`, - `condition`...). We are going to make some user unhappy if they cannot - have an `external` directory at the top of their workspace (I would - just not use a build system that goes against my workspace structure). - While it is still time to do it, we should rename them with a nice - naming convention. A good way to do it is to make top-package name - constants, possibly settable in the WORKSPACE file (so we can actually - keep the name we like but user that are bothered by that can change - it). - 4. As we will remove the tools directory from the workspace, it - makes sense to add another prelude_bazel file somewhere else. As the - `WORKSPACE` file controls the workspace, it makes sense to have the - `prelude_bazel` logic in it (and the load statement should support - labels so that a user can actually specify remote repository labels). - -## Appendix B. Skylark-like code examples of configuration functions - -This is just a quick draft, please feel free to propose improvements: - -```python -# env is the environment, attrs are the values set either from the command-line -# or from loading the package -def autodetect_method(env, attrs): - if not attrs.java_version: # If not given in the command line nor loaded - attrs.java_version = 8 - if not attrs.jdk_path: - if env.has("JDK_HOME"): - attrs.jdk_path = env.get("JDK_HOME") - elif env.os = "darwin": - attrs.jdk_path = system("/usr/libexec/java_home -v 1." + attrs.java_version + "+") - else: - attrs.jdk_path = basename(basename(readlink(env.path.find(java)))) - if not attrs.jdk_path: - fail("Could not find JDK home, please set it with `bazel init java:jdk_path=/path/to/jdk`") - attrs.__some_other_path = first(glob(["/usr/bin/java", "/usr/local/bin/java"])) - - -# attrs is the list of attributes. It basically contains the list of rules -# we should generate in the corresponding package. Please note -# That all labels are replaced by relative ones as it should not be able -# to write out of the package. -def generate_method(attrs): - scratch_file("BUILD.jdk", """ -Content of the jdk BUILD file. -""") - # Create binding using local_repository. This should not lie in - # the WORKSPACE file but in a separate WORKSPACE file in the hidden - # directory. - local_repository(name = "jdk", path = attrs.jdk_path, build_file = "BUILD.jdk") - bind("@jdk//jdk", "jdk") # also add a filegroup("jdk", "//external:jdk") - java_toolchain(name = "toolchain", source = attrs.java_version, target = attrs.java_version) - # The magic __BAZEL_*__ variable could be set so we don’t - # redownload the repository if possible. This install_target - # should leverage the work already done on remote repositories. - # This should build and copy the result into the tools directory with - # The corresponding exports_files now. - install_target(__BAZEL_REPOSITORY__, __BAZEL_VERSION__, "//src/java_tools/buildjar:JavaBuilder_deploy.jar") - install_target(__BAZEL_REPOSITORY__, __BAZEL_VERSION__, "//src/java_tools/buildjar:JavaBuilder_deploy.jar") - copy("https://ijar_url", "ijar") - - - -# Load the package attributes. -# - attrs should be written and value will be replaced by the user-provided -# one if any -# - query is a query object restricted to the target package and resolving label -# relatively to the target package. This object should also be able to search -# for repository binding -# Note that the query will resolve in the actual tools directory, not the hidden -# one if it exists whereas the generation only happens in the hidden one. -def load_method(attrs, query): - java_toolchain = query.getOne(kind("java_toolchain", "...")) - if java_toolchain: - attrs.jdk_version = max(java_toolchain.source, java_toolchain.target) - jdk = query.getOne(attr("name", "jdk", kind("local_repository", "..."))) - if jdk: - attrs.jdk_path = jdk.path -``` diff --git a/site/designs/_posts/2015-07-02-skylark-remote-repositories.md b/site/designs/_posts/2015-07-02-skylark-remote-repositories.md deleted file mode 100644 index 9038df85a5..0000000000 --- a/site/designs/_posts/2015-07-02-skylark-remote-repositories.md +++ /dev/null @@ -1,167 +0,0 @@ ---- -layout: contribute -title: Skylark Remote Repositories ---- - -# Design Document: Skylark Remote Repositories - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status:** implemented - -**Author:** dmarting@google.com - -**Reviewers:** lberki@google.com, laurentlb@google.com, kchodorow@google.com - -**Design document published**: 02 July 2015 - -## Context - -[Skylark](/docs/skylark/index.html) is the -extension language for Bazel and lets Bazel users describe the -build for new languages easily. External users do not create -native rules and we want to avoid them doing so. - -[Remote repositories](/docs/external.html) -are a convenient way to specify your third party dependencies -and to fetch them along with the build if you don’t want to -check them in your repository. - -This document discuss “Skylark Remote Repositories”, that is -creating new remote repository rules using Skylark. - -## Why? - - - Enable users to specify new kind of repositories, we already have - requests for PyPI for example. We don’t want to be supporting every - kind of repository that exists outside. - - Enable Skylark rules to write macros to have one-liners for - including all their tools in your WORKSPACE file. - - Enable configuration of languages tooling: - [bazel init](/design/2015/03/06/bazel-init.html) - first approach as a separate tools is not really user-friendly and - the same kind of flexibility can be achieved by creating - repositories rule in Skylark. An example for the JDK is - [here](https://github.com/damienmg/bazel/commit/c40bd9c71965f0a8c85b732a0b0c3dbe31e017cf/jdk.bzl). - -## User interface (see the [JDK example](https://github.com/damienmg/bazel/commit/c40bd9c71965f0a8c85b732a0b0c3dbe31e017cf/jdk.bzl)) - -The load statement will now be available from the WORKSPACE, working -the same way it does for build file but with WORKSPACE specific -functions instead. - -In the same way that we have macros and rules for the BUILD file, we -are going to have macros and rule for the WORKSPACE file. The former -will be a convenient way to combine remote repositories and the latter -enable creation of new repositories kind. - -### Macros - -Skylark macros would be activated in the WORKSPACE file and would -behave as expected. Macros would enable to combine remote repositories -creation and bind into a single rules E.g. `setup_java()` would set-up -all bindings and the local repository needed to build java target: - -```python -def setup_java(): - native.new_local_repository(name = “jdk-local”, path = “/usr/share/java/jdk8”, build_file = “jdk.BUILD”) - for target in ["jni_header", "jni_md_header", "langtools", "bootclasspath", "extdir", "toolchain", "jdk", "java", "javac", "jar"]: - native.bind(name=target, actual="@%s//:%s" % (name, target)) - native.bind(name="jni_md_header-linux", actual="@%s//:jni_md_header" % name) - native.bind(name="jni_md_header-darwin", actual="@%s//:jni_md_header" % name) -``` - -### Remote repository rule -A remote repository rule would be set-up the same way we set-up a -build rule but with the `repository_rule` statement: - -```python -jdk_repository = repository_rule( - implementation = my_impl, - attrs = { - “java_home”: attr.string(mandatory=False), - “java_version”: attr.string(default=“1.8”), - } -``` - -This statement takes only 2 arguments: an implementation function and -a list of attributes. The syntax is similar to the rule statement but -attributes can only takes primitive type (String, Label, Integer, -Boolean, …) and not artifacts. - -The implementation function takes exactly one argument: the repository -context. This context will provides many convenience methods for doing -non hermetic operations, e.g., : - -- For discovering the environment: - * access system environment (`ctxt.os`), - * execute a program and get the standard output (`ctxt.execute`) to - discover the environment, - * … -- For creating the remote repository: - * fetch an artifact from URL (`ctxt.download`), - * uncompress an artifact (`ctxt.path(...).uncompress(outputPath)`), - * “copy” a directory from the system (`ctx.fetch_path(...)`), - * create a build file (`ctxt.build_file(...)`) - * ... - -The precise list of methods the repository context will support will -be augmented on-demand depending on what makes sense for our users. - -## How? - -A preliminary quick and dirty prototype can be found -[here](https://github.com/damienmg/bazel/commit/3a483ab53e547ae677599215faa9e3398bbd06ba) -and -[here](https://github.com/damienmg/bazel/commit/c40bd9c71965f0a8c85b732a0b0c3dbe31e017cf). - -Here what the prototype does: - - 1. [First commit](https://github.com/damienmg/bazel/commit/3a483ab53e547ae677599215faa9e3398bbd06ba) - activate Skylark macros and repositories - a. Allow Skylark load statements in the WORKSPACE file by adding - the various hook and a WorkspaceContext. - b. A new repository_rule in Skylark that can be called only from - the WORKSPACE file. - c. A new repository context that is passed to repository rule and - that should contain all the non-hermetic stuff so the rest of - skylark stays hermetic. - d. A bit of hack for tweaking the SkylarkNativeModule when in - WORKSPACE file to comply with the structure of the WORKSPACE - rules. - e. A dirty hack to load the SkylarkRepositoryFunction as a Skylark - module without breaking the package boundaries. This is due of - technical debts on loading Skylark module nicely (there is a TODO - to do it correctly). - 2. [Second commit](https://github.com/damienmg/bazel/commit/c40bd9c71965f0a8c85b732a0b0c3dbe31e017cf) - showcase the usage of Skylark remote repositories as a configuration step. - a. Add an example for fetching JDK dependencies. It does both the - detection and the fetching. - b. Add the necessary methods in the SkylarkRepositoryContext for - making the example work. - c. Added the search method to the Skylark string object (to do a - regex search). - -## Roadmap - -The obvious choice for the roadmap is to remake all those works, -correctly commented and tested, and then add methods to the -SkylarkRepositoryContext for full support. -More precisely the correct order of the work should be: - -1. Activate Skylark Macros taking part of 1.a and doing correctly 1.d - [[DONE](https://github.com/bazelbuild/bazel/commit/8ca065c2ae941309b53f95313a68e585ecc70560)] -2. Fix Skylark module load (1.e) - [[DONE](https://github.com/bazelbuild/bazel/commit/f81c675928c6beeaae5f66480dc7dbef47f75fb8)] -3. Add the SkylarkRepositoryFunction and empty context (1.b and 1.c) - [[DONE](https://github.com/bazelbuild/bazel/commit/653df8813dd74042e8e084eeae238a8b9f16a3ca)] -4. [Extends SkylarkRepositoryContext for handling C++ configuration](/designs/2016/02/16/cpp-autoconf.html) - [[DONE](https://github.com/bazelbuild/bazel/commit/8fa5ae6a6364100f2a7f9130e62eb0edb447339a)] -5. Extends SkylarkRepositoryContext - [[DONE](https://github.com/bazelbuild/bazel/commit/97f2c842ae706eccd822d8d264aba66c6102bade)] - for handling PyPI -6. Document - [[DONE](/docs/skylark/repository_rules.html)] -7. Extends SkylarkRepositoryContext for handling Docker pull diff --git a/site/designs/_posts/2016-02-16-cpp-autoconf.md b/site/designs/_posts/2016-02-16-cpp-autoconf.md deleted file mode 100644 index de42e17406..0000000000 --- a/site/designs/_posts/2016-02-16-cpp-autoconf.md +++ /dev/null @@ -1,151 +0,0 @@ ---- -layout: contribute -title: Generating C++ crosstool with a Skylark Remote Repository ---- - -# Design Document: Generating C++ crosstool with a Skylark Remote Repository - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status:** implemented - -**Author:** dmarting@google.com - -**Reviewers:** lberki@google.com - -**Design document published**: 16 February 2016 - -## Context - -[Skylark](/docs/skylark/index.html) is the -extension language for Bazel and lets Bazel users describe the -build for new languages easily. External users do not create -native rules and we want to avoid them doing so. - -[Remote repositories](/docs/external.html) -are a convenient way to specify your third party dependencies -and to fetch them along with the build if you don’t want to -check them in your repository. - -[Skylark remote -repositories](/designs/2015/07/02/skylark-remote-repositories.html) is -an ongoing effort to support specifying new remote repositories using -Skylark. - -## Why? - -Configurability issues are stopping users from compiling and using -Bazel on complex setup. In particular, -[TensorFlow](https://tensorflow.io)’s users runs on -various hardware where gcc is installed on non-standard directory that -needs to change the -[CROSSTOOL](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/tools/cpp/CROSSTOOL) -file (see -[issue #531](https://github.com/bazelbuild/bazel/issues/531)). -This generally requires to change the list of include directories, the -path to gcc and sometimes also the linking option to find the correct -libraries at runtime. Some platform even requires -[special wrappers around gcc](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/tools/cpp/osx_gcc_wrapper.sh). - -Java solved the problem by setting a custom repository -([@local_jdk](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/src/main/java/com/google/devtools/build/lib/bazel/rules/java/jdk.WORKSPACE#L3)) -where the path is automatically detected using the [location of the JVM -running -Bazel](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactory.java#L414). -But this approach does not scale well with complex language setup like -C++. - -We reduced the number of C++ issues the user had with removing all C++ -compilation in the bootstrap of Bazel. However, to properly handle -those platform, Bazel needs some level of auto-configuration -([Kythe](https://github.com/google/kythe/blob/a29f0adc6fa11550f66bc2278f17b89b9e02de18/setup_bazel.sh) -and -[Tensorflow](https://github.com/tensorflow/tensorflow/blob/a81c4f9cd01563e97fc6f179e4d70960fc9b02ae/configure) -have their own auto-configuration scripts). This document discuss how -to use a skylark remote repository to implement a simple -auto-configuration for C++ crosstool (step 4 of the roadmap from the -[Skylark remote -repositories](/design/2015/07/02/skylark-remote-repositories.html) -document). - -## C++ toolchain detection - -Until now here the various issues user have faced using a custom C++ -toolchain: - - 1. C++ compiler is not at the expected location. - 2. C++ compiler is `clang` and not `gcc` or behaves differently than - what Bazel C++ rules expect. - 3. Libraries are not in the default location. - 4. Headers are not in the default location. - 5. Path of libraries or headers are outside of the default mounted - paths. - -The current fix we propose to the user for the various issue are: - - 1. Change the tool paths in - [tools/cpp/CROSSTOOL#L87](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/tools/cpp/CROSSTOOL#L87). - 2. Add a wrapper like - [tools/cpp/osx\_gcc\_wrapper.sh](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/tools/cpp/osx_gcc_wrapper.sh) - and modify some options from the CROSSTOOL file. - 3. Add `-Wl,rpath,` option to the - [linker\_flags](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/tools/cpp/CROSSTOOL#L93). - 4. Add the directories to the - [cxx\_builtin\_include\_dirs](https://github.com/bazelbuild/bazel/blob/6b6ff76945c80fb8b11b71d402b5146c85b86859/tools/cpp/CROSSTOOL#L100). - 5. Deactivate sandboxing - -To address those issues, we propose to add the followings methods to -the repository context object: - - 1. `which(cmd)` returns the path to the binary designed by `cmd`, - looking for it in the path environment variable (or equivalent). - This will help getting the path to the C++ compiler. - 2. `execute([arg0, arg1, ..., argn])` executes a command and returns an - `exec_result` struct containing: - * `stdout` the content of the standard output, - * `stderr` the content of the standard error output, and - * `return_code` the return code of the execution. - Executing `$(CC)` will help detect whether we are using gcc or - clang. - 3. An `os` object with an environ map containing the list of - environment variable. The os object will be extended to - contains all OS specific variables (platform name and much more). - 4. `execute([..])` from 2 will be used to run [`gcc ... --v`](http://stackoverflow.com/questions/11946294/dump-include-paths-from-g) - to list the built-in include directories. -To address the issue 5, we can add the list of paths to dependencies to the -[crosstool rule in the BUILD -file](https://github.com/bazelbuild/bazel/wiki/Building-with-a-custom-toolchain). - -## Writing the cpp package - -Once we have resolved all the information from the system, we need to -write two or three files: - - - The `BUILD` file that will contains the corresponding - `cc_toolchain` rules - - The `CROSSTOOL` file - - Optionally, the wrapper script. - -We should extends the context with a `file(path, content)` method, where -path is a path relative to the repository root and content the content -of the file to write. - -To ease the writing of crosstool, we should also provide a -`template(path, label, variables)` method which will write the file -pointed by path using the file pointed by label (should be a -FileValue) in which variables have been replaced in the same way that -[template_action](http://bazel.build/docs/skylark/lib/ctx.html#template_action) -works. - -## Rollout plan - -The implementation plan would be: - - 1. Implements `which`, `execute`, `os`, `file` and `template` - [__DONE__] - 2. Write the `cc_configure` repository rule which does the work. Use - GitHub bugs as inputs on which platform to support. [__DONE__] - 3. Advertise the existence of `cc_configure` [__DONE__] diff --git a/site/designs/_posts/2016-05-23-beautiful-error-messages.md b/site/designs/_posts/2016-05-23-beautiful-error-messages.md deleted file mode 100644 index e8370e1a5a..0000000000 --- a/site/designs/_posts/2016-05-23-beautiful-error-messages.md +++ /dev/null @@ -1,362 +0,0 @@ ---- -layout: documentation -title: Beautiful error messages ---- - -# Design Document: Beautiful error messages - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Reviewed, not yet implemented - -**Author**: laurentlb@google.com - -**Reviewers**: ulfjack@google.com - -**Design document published**: 23 May 2016 - -**Related**: [Implementing beautiful error -messages](/designs/2016/05/26/implementing-beautiful-error-messages.html) - -## Background - -Providing good error messages is important for usability. It is especially -important for new users who try to understand a tool. It can also drive adoption -(as in clang vs gcc). - -I suggest we make error messages in Bazel more helpful. - -## Other tools - -Screenshots below are from Elm and Clang, both have been praised for the quality -of their diagnostics. They are good examples we can draw our inspiration from. - -See also [Compiler Errors for Humans] -(http://elm-lang.org/blog/compiler-errors-for-humans), a blog article about the -work that went into Elm. - - - - - - - - - -## What we can improve - -* Users need **context**: Showing exact code in error is useful (with a caret - ^). Currently location in Bazel error messages is often approximative (e.g. - column number is often off, we show location of a rule instead of the - specific attribute). -* **Colors** make messages easier to understand. -* **Suggestions** are useful. Bazel should try to guess what the user wanted - to do, and suggest a fix. A typical example is to detect typos. -* A **link** can give more more information (e.g. elm can show a link to - https://github.com/elm-lang/elm-compiler/blob/master/hints/recursive-alias.md). - If you don’t, users will have to copy error messages and look up on search - engine. - -We have some limitations though: - -* Due to memory constraints, we cannot keep the source file or even the AST - after loading phase. Maybe code printing can be limited to errors in loading - phase? -* Code generated through macros can make error reporting more difficult. - -## Examples - -In each example below, you’ll find 1. the input code, 2. the current output from -Bazel, 3. a suggested improvement. - -These are just suggestions we can iterate on. - -*** - -Input: -<pre><code>my\_obj = select({ - ":something": [1], - "other": [2], -}) -t = [x for x in my\_obj] -</code></pre> - -Current: -<pre><code>ERROR: /path/BUILD:6:5: type 'select' is not iterable -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:6:16: <strong>my\_obj of type 'select' is not iterable.</strong> You can iterate only on string, lists, tuples, or dicts. -t = [x for x in my\_obj] - ^----- -Related documentation: http://www.bazel.build/docs/be/functions.html#select -</code></pre> - -*** - -Input: -<pre><code>t = [x for x in] -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:16: syntax error at ']': expected expression -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:1:16: <strong>Syntax error: expected expression, got ']'.</strong> -t = [x for x in] - ^ -</code></pre> - -*** - -Input: -<pre><code>glob(["*.cc"], excludes = ["foo.cc"]) -</code></pre> - -Current: -<pre><code>ERROR: /path/BUILD:1:1: unexpected keyword 'excludes' in call to glob(include: sequence of strings, exclude: sequence of strings = [], exclude\_directories: int = 1) -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:1:5: <strong>'excludes' is an invalid keyword argument for the function glob(include, exclude, exclude\_directories).</strong> Did you mean exclude? -glob(["*.cc"], excludes = ["foo.cc"]) - ^------- - exclude -Related documentation: http://www.bazel.build/docs/be/functions.html#glob -</code></pre> - -*** - -Input: -<pre><code>cclibrary(name = "x") -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:1: name 'cclibrary' is not defined -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:1:1: <strong>Name 'cclibrary' is not defined.</strong> Did you mean cc\_library? -cclibrary(name = "x") -^-------- -cc\_library -</code></pre> - -*** - -Input: -<pre><code>cc\_library( - name = "x", - deps = ":lib", -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:1: //test:x: expected value of type 'list(label)' for attribute 'deps' in 'cc\_library' rule, but got ":lib" (string) -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:3:5: <strong>Expected value of type 'list(string)' for attribute 'deps' in 'cc\_library' rule, but got ":lib" (string). </strong>Did you mean [":lib"]? - deps = ":lib", - ^----- - [":lib"] -Related documentation: http://www.bazel.build/docs/be/c-cpp.html#cc\_library -</code></pre> - -*** - -Input: -<pre><code>VAR = ":lib" -cc\_library( - name = "x", - deps = VAR, -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:3:1: //test:x: expected value of type 'list(label)' for attribute 'deps' in 'cc\_library' rule, but got ":lib" (string) -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:5:5: <strong>Expected value of type 'list(string)' for attribute 'deps' in 'cc\_library' rule, but got ":lib" (string).</strong> Did you mean [VAR]? - deps = VAR, - ^-- - [VAR] -Related documentation: http://www.bazel.build/docs/be/c-cpp.html#cc\_library -</code></pre> - -*** - -Input: -<pre><code>cc\_library( - name = "name", - deps = ["/test/foo.cc"], -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:1: //test:name: invalid label '/test/foo.cc' in element 0 of attribute 'srcs' in 'cc\_library' rule: invalid target name '/test/foo.cc': target names may not start with '/' -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:3:13: <strong>Invalid label '/test:foo.cc' in deps. Labels relative to the root start with //.</strong> Did you mean '//test:foo.cc'? - deps = ["/test:foo.cc"], - ^------------- - "//test:foo.cc" -Related documentation: http://www.bazel.build/docs/build-ref.html#labels -</code></pre> - -*** - -Input: -<pre><code>cc\_library( - name = "name", - srcs = [":x"], -) -genrule( - name = "x", - outs = ["file.ext"], - cmd = "touch $@", -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:3:12: in srcs attribute of cc\_library rule //test:name: '//test:x' does not produce any cc\_library srcs files (expected .cc, .cpp, .cxx, .c++, .C, .c, .h, .hh, .hpp, .hxx, .inc, .S, .s, .asm, .a, .pic.a, .lo, .pic.lo, .so, .dylib, .o or .pic.o) -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:3:12: <strong>In srcs attribute of //test:name (cc\_library), '//test:x' does not produce any cc\_library srcs files</strong> (expected extension .cc, .cpp, .cxx, .c++, .C, .c, .h, .hh, .hpp, .hxx, .inc, .S, .s, .asm, .a, .pic.a, .lo, .pic.lo, .so, .dylib, .o or .pic.o). <strong>Target //test:x (genrule) generated 'file.ext'</strong>. -Related documentation: http://www.bazel.build/docs/be/c-cpp.html#cc\_library -</code></pre> - -*** - -Input: -<pre><code>cc\_library( - name = "name", - deps = ["//base:scheduling\_domain-test"], -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:1: in cc\_library rule //test:name: non-test target '//test:name' depends on testonly target '//base:scheduling\_domain-test' and doesn't have testonly attribute set -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:3:5: <strong>In deps attribute of //test:name (cc\_library), '//base:scheduling\_domain-test' (cc\_library) is marked as testonly.</strong> You may either add: - testonly = 1 -to //test:name definition, or remove testonly from //base:scheduling\_domain-test, or remove the dependency. -Related documentation: http://www.bazel.build/docs/be/common-definitions.html#common.testonly -</code></pre> - -*** - -Input: -<pre><code>cc\_library( - name = "name", - srcs = ["//base:arena.cc"], -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:1: Target '//base:arena.cc' is not visible from target '//test:name'. Check the visibility declaration of the former target if you think the dependency is legitimate -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:3:5: <strong>In srcs attribute of //test:name (cc\_library), '//base:arena.cc' (file) is not visible.</strong> You may change the visibility of the file using exports\_file, or expose the file via a library rule, or remove the dependency. -//base:arena.cc has currently private visibility. -Related documentation: http://www.bazel.build/docs/be/common-definitions.html#common.visibility -</code></pre> - -*** - -Input: -<pre><code>cc\_binary( - name = "bin", - deps = [":lib"], -) -cc\_library( - name = "lib", - srcs = [":src"], -) -genrule( - name = "src", - outs = ["file.cc"], - cmd = "touch $@", - tools = [":bin"], -) -</code></pre> -Current: -<pre><code>ERROR: /path/BUILD:1:1: in cc\_binary rule //test:bin: cycle in dependency graph: - //test:bin - //test:lib - //test:src - \* //test:bin (host) - //test:lib (host) - //test:src (host) - \* //test:bin (host) -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:1:1: <strong>Cycle in dependency graph detected:</strong> - cc\_binary //test:bin depends on (via deps): - cc\_library //test:lib depends on (via srcs): - genrule //test:src depends on (via tools): - <strong>\* cc\_binary //test:bin</strong> [host] depends on (via deps): - <strong>| cc\_library //test:lib</strong> [host] depends on (via srcs): - <strong>| genrule //test:src</strong> [host] depends on (via tools): - <strong>* cc\_binary //test:bin</strong> [host] -Please modify at least one of the dependencies to break the cycle. -</code></pre> - -*** - -Input: - -ext.bzl -<pre><code>def foo(name): - native.genrule( - name = name, - outs = ["file.cc"], - cmd = "touch $@", -) -</code></pre> -BUILD -<pre><code>load(":ext.bzl", "foo") -foo("src2") -files = ["file.cc"] -genrule( - name = "src", - outs = var, - cmd = "touch $@", -) -</code></pre> -Current: -<pre><code>ERROR:/path/BUILD:7:1: generated file 'file.cc' in rule 'src' conflicts with existing generated file from rule 'src2' -</code></pre> -Suggested: -<pre><code>ERROR: /path/BUILD:7:1: <strong>Generated file 'file.cc' in rule 'src' conflicts with existing generated file from rule 'src2'.</strong> -'src' is defined line 7: - genrule( - name = "src", - outs = ["file.cc"], - cmd = "touch $@", - ) -'src2' is generated by the function foo (line 3) and is equivalent to: - genrule( - name = "src2", - outs = ["file.cc"], - cmd = "touch $@", - ) -</code></pre> - -[Another case](https://github.com/bazelbuild/bazel/issues/1307) - -## Suggestions - -* Symbol not found during evaluation - * Suggest another symbol from the environment -* Invalid label (parse error) - * Link to label documentation, suggest a fix -* Directory of the label doesn’t exist - * Look in file system and suggest another directory -* Label name not found - * Suggest another name, from the same package -* Keyword argument doesn’t exist - * Suggest name, based on function signature -* Field not found (obj.nonexistent) - * Suggest name, based on list of fields - -## Action items - -* Show context line + carret -* Suggest spelling fixes -* Show documentation links -* Show expanded rules (for action conflicts or errors during analysis phase) -* Improve error messages for rules developers (e.g. pretty-print action - graph?) -* Review existing error messages + add ad-hoc suggestions diff --git a/site/designs/_posts/2016-05-26-implementing-beautiful-error-messages.md b/site/designs/_posts/2016-05-26-implementing-beautiful-error-messages.md deleted file mode 100644 index c07c665805..0000000000 --- a/site/designs/_posts/2016-05-26-implementing-beautiful-error-messages.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -layout: contribute -title: Implementing Beautiful Error Messages (Loading Phase) ---- - -# Design Document: Implementing Beautiful Error Messages (Loading Phase) - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: unimplemented - -**Author**: [laurentlb@google.com](mailto:laurentlb@google.com) - -**Design document published**: 26 May 2016 - -**Related**: ["Beautiful error messages"](/designs/2016/05/23/beautiful-error-messages.html) - -## Introduction - -This is a followup to the document ["Beautiful error messages"](/designs/2016/05/23/beautiful-error-messages.html). - -The purpose of this document is to outline a design for some plumbing that will -allow the sort of errors described in that document to be emitted by Blaze for -loading time `BUILD` file errors. - -## Review: What needs to be done - -In the ["Beautiful error messages"](/designs/2016/05/23/beautiful-error-messages.html) -document, four characteristics of error messages -are enumerated: - -1. **Context**: The erroneous text in the `BUILD` file in question is shown, - with a caret pointing at the exact expression in question. - -2. **Colors**: Everyone loves colors. - -3. **Suggestions**: A guess of how the error should be fixed is shown. - -4. **Links**: Documentation is referenced directly in error messages. - -This document covers (1) and (3). - -## Current Error Infrastructure - -In the loading phase, Bazel parses `BUILD` files into a tree of [ASTNode] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/syntax/ASTNode.java) -instances, with a [BuildFileAST] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/syntax/BuildFileAST.java) -at the root and `Statement` instances at the leaves. Each statement implements -doExec, which can throw an [EvalException] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/syntax/EvalException.java). -That exception is translated to the error printed to the terminal. - -An `EvalException` encapsulates the information given in an error. Fleshing out -the contents of this exception type is a good starting point for implementing -new error features. - -## Implementation: Context - -An `EvalException` contains a reference to a `Location`, which encapsulates the -line/character information currently specified in a Bazel error. - -Generally, an `ASTNode` will have a location instance that is populated by the -parser as it moves from token to token ([example] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/syntax/Parser.java#L607)). -Right now, since the location only contains pure syntactic information, the -parser [calls into the lexer] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/syntax/Parser.java#L413) -to create the location instance. While the parser perhaps *could* also encode -AST information into the `Location`, that shouldn’t be necessary to provide -"context" in the form of a printed line and a carat. It seems that the -lexer maintains `BUILD` file info as a buffer, and should be able to -parameterize `Location` instances with the actual contents of the line in -question. If this is true, then implementing (1) above shouldn’t involve much -more than fleshing out [LocationInfo] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/syntax/Lexer.java#L72). - -## Implementation: Suggestions - -While data tracked by the lexer should be sufficient to encode an offending -line from a `BUILD` file into an error, providing suggestions will probably -require semantic information only retrievable from the parsed AST. Furthermore, -we require a mechanism in `EvalException` to perform computation on AST data in -order to generate suggestions. - -It seems an unlikely solution to encode AST information in `Location` instances, -since those instances are produced by the parser before the AST, or even the AST -node in question, is necessarily complete. - -Instead, here are a couple proposals: - -1. An abstract subclass of `EvalException` (e.g. `ContextualEvalException`) -that knows how to create an error message with suggestions given unimplemented -suggestion generation logic. - -2. A further group of exceptions that are parameterized with a particular sort -of `ASTNode` and know how to generate suggestions. As an example, this -exception type could report typos in rule names. - -```java -public class IdentifierEvalException extends ContextualEvalException { - public static final Map<String, String> TYPOS = Map.of( - "cclibrary", "cc_library", - ect... - ) - public IdentifierEvalException(Identifier identifier, Location loc) {...} - @Override - protected Suggestion generateSuggestion() { - if (TYPOS.keySet().contains(identifier.getName()) { - return TypoSuggestion(identifier, TYPOS.get(identifier.getName())); - } - } -} -``` - -Once the plumbing around (1) is in place, we can add subclasses at our leisure -to provide suggestions. - -Furthermore, a `ContextualEvalException` can be made to have enough information -to not only provide suggestions, but also to return context-aware error -messages. Consider this example, from the -["Beautiful error messages"](/designs/2016/05/23/beautiful-error-messages.html) -document: - -```python -my_obj = select({ - ":something": [1], - "other": [2], -}) - -t = [x for x in my_obj] -``` - -<pre> -<font color="red">ERROR:</font> /test/BUILD:6:5: type 'select' is not iterable. -</pre> -<pre> -<font color="red"> -ERROR: /test/BUILD:6:16:</font> <b>my_obj of type 'select' is not iterable.</b> -You can iterate only on strings, lists, tuples or dicts. -t = [x for x in my_obj] - ^ -Related documentation: http://documentation#select -</pre> - -We can imagine a `NotIterableEvalException` that knows not only about the type -`select`, but is also parameterized with the erroneous expression ```my_objc```. - -## Problem: Serialization - -The above proposal hinges on the ability to store a file pointer in the -`Location` object, to be dereferenced at error-time to obtain the entire `BUILD` -file. This opens the door to some issues: - -1. A `Location` instance is stored for every node in the parse tree of every -`BUILD` file. Even a file pointer in each `Location` may have substantial -memory/speed impact. -<p>This impact is easily measurable and likely tolerable in order to achieve -better error messages. However, it is clear that storing anything much larger -than a file pointer (like a fragment of the file, or the file itself) in each -`Location` would be untenable. - -2. The `Location` object is serialized, since the AST is part of a `SkyValue`. -This pointer, then, must be serializable. -<p>This in particular is troubling because, unlike the java heap in a local -Bazel execution, the `SkyValue` containing the AST does not necessarily have the -`BUILD` file. However, it is not clear to me that this means the file must be -copied into each `Location` object. The AST presumably resides in a single -`SkyValue` - one copy of the file in that `SkyValue`, with a pointer to that -file in the `Location`, would be sufficient, it seems. -<p>The nature of the `SkyValue` that contains that AST must be determined, -thinking about if and how to embed the `BUILD` file into that `SkyValue`, and -strategizing about a good serialization for a `Location` object that -contains a file pointer. diff --git a/site/designs/_posts/2016-06-02-sandboxing.md b/site/designs/_posts/2016-06-02-sandboxing.md deleted file mode 100644 index 579b115fff..0000000000 --- a/site/designs/_posts/2016-06-02-sandboxing.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -layout: contribute -title: Sandboxing ---- - -# Design Document: Bazel Sandboxing 2.0 - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status:** launched - -**Author:** [philwo@google.com](mailto:philwo@google.com) - -**Design document published**: 02 June 2016 - -The section "Handling of environment variables" inspired and was then -superseded by the more detailed -[Specifying environment variables](/docs/designs/2016/06/21/environment.html) -design document. - -## Current situation - -Tools that use undeclared input files (files that are not explicitly listed in -the dependencies of an action) are a problem, as Bazel cannot keep track of them -and thus they can cause builds to become incorrect: When one of the undeclared -input files changes, Bazel will still believe that the build is up-to-date and -won't rebuild the action - resulting in an incorrect incremental build. - -Bazel uses sandboxing to prevent tools (e.g. compilers, linkers, ...) from -accidentally working with input files that are not a declared dependency of an -action - the idea is to run each tool in an environment that contains only the -explicitly declared input files of the action. Thus, there simply are no other -files that a tool could access. - -In theory this works well, but as nearly all Bazel users rely at least on some -tools provided by their operating system (e.g. `/usr/bin/zip`, `/usr/bin/gcc`), -which in turn require shared libraries, helper tools or data from other parts -of the installed OS, Bazel currently mounts a number of hard-coded directories -from the operating system into the sandbox in addition to the explicitly -declared inputs. - -However, even with that some users continue to run into issues, making Bazel -hard to use - e.g. the compiler they want to use is in a directory that's not -part of the hard-coded list (such as `/usr/local` or `/opt`) or the tool needs -access to device files (e.g. the nVidia CUDA SDK). - -## Proposal - -We think that it's time to revisit how we do sandboxing in the default settings -of Bazel. Sandboxing was intended to protect the user from forgetting to -declare explicit dependencies between their targets and to protect from tests -or tools accidentally writing all over the hard-disk (e.g. a test that wants to -clean up its temporary work directory via rm -rf and unfortunately wipes the -whole disk), not so much for protecting against an operating system having any -influence on the build. For these users, the current sandboxing with its -hard-coded list of allowed directories is too strict. - -On the other hand, some people absolutely do want 100% reproducible and -hermetic builds - and for them the current sandboxing actually isn't strict -enough, as it allows access to various files from the operating system. - -We believe we have found a solution that satisfies the demands of all users: - - * Bazel sandboxing will by default recursively mount the root directory `/` - into each sandbox in read-only mode, excluding the workspace directory (so - that source files cannot be read from that well-known path) and with a new - empty, writable execroot that contains the declared inputs of the action. - * In addition, Bazel will allow to mount a 'base image' or 'base directory' as - the root directory of the sandbox, thus completely removing any connection - to the operating system the user is running Bazel under. For example, a - project might decide that all builds should be done inside a standardized - Ubuntu 16.04 LTS environment containing certain versions of gcc, etc., that - is shipped as a base image. Now, even if the developer uses Arch Linux or - CentOS on their machine, they can build using the same environment as - everyone else, thus getting the exact same and reproducible outputs. - -### Base images - -Base images are simply `.tar.gz`'s of a directory structure that contains all -files necessary to execute binaries in, e.g. the output of “debootstrap” or -what you would usually “chroot” in and then run a tool inside. They should be -referred to via labels and could for example be downloaded from somewhere via -a `http_file` rule in the WORKSPACE. - -We're investigating if we can reuse -[Docker images (OCI)](https://github.com/opencontainers/image-spec/blob/v0.1.0/serialization.md) -for this, which would make it easier for users to get started with this -feature. - -### Handling of environment variables - -As part of this project, we also propose to change the handling of environment -variables (e.g. `PATH`) in Bazel, as we believe they are an important part of -the configuration of the environment that the build runs in. - -As an example, Bazel currently [resets PATH to a hard-coded string] -(https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelConfiguration.java), -which may not be suitable for the environment that it actually runs in - e.g. -if a user installs a tool called `babel` in `/usr/local/bin` and they call -`babel` in a shell script or Skylark rule they wrote, [they expect it to just -work] (https://github.com/bazelbuild/bazel/issues/884). We can argue that they -instead should check in their tool to the repository and not rely on `PATH` -lookup to find it, however this is sometimes not possible due to: - - * Users just don't think it's feasible and instead want to take whatever is - installed on the system, - * Bazel's restrictions in valid package label identifiers ([you can't check in - nodejs](https://github.com/bazelbuild/bazel/issues/884#issuecomment-183378680) - into your repository or even make it part of a filegroup, because it - contains files that have characters like `$` that are currently illegal from - Bazel's point of view, though that may change in the future), - * Licensing restrictions that disallow users checking in certain tools (such - as XCode). - -The proposal how Bazel should decide whether an environment variable should be -included in the environment of a Spawn is: - - * If `use_default_shell_env` is `True`, set `PATH` and `TMPDIR` env vars - (as we currently do). - * If a rule declares its need for an environment variable, take it. - * We already have an [“env” attribute in Skylark actions] - (http://www.bazel.build/docs/skylark/lib/ctx.html#action) that allows one to - set variables to hard-coded strings, we have `use_default_shell_env` in - Skylark actions, which pulls in `PATH` and `TMPDIR`, but we don't have any - way to just say "This rule needs this environment variable". Laurent - suggested that we discuss this later, as adding yet another attribute is - annoying - maybe there's some way we can fold all these use cases into one - attribute. - * We might want to add the same attribute to genrule as well then. - * Don't include any other environment variables. - -If Bazel decided that an environment variable is needed by a rule, the next -step is to figure out its value. The proposal how Bazel should decide the value -of an environment variable is: - - * If an environment variable is overridden in the `WORKSPACE.local` file - ("machine-specific settings"), take it from there. - * If an environment variable is overridden in the `WORKSPACE` file - ("project-specific settings"), always take the value from there. - * If not and we use a base image, take the environment variable from its - specification (as in OCI). - * If not, take it from the user's environment. - -If an environment variable that is used by a rule changed compared to when it -was built last time, its target has to be rebuild for correctness. - -Bazel should instead use `PATH` from the environment and for correctness -trigger a rebuild when it changes. - -*Open question: Should the whitelist of environment variables be configurable, -e.g. in the WORKSPACE file?* - -### What files does a sandboxed process have access to? - -Ideally, we would want to execute SpawnActions in an environment that looks like this: - - * Allows read access to everything in /. - * Except the workspace (e.g. /home/philwo/src/bazel). - * Except the "real" execroot (e.g. /tmp/_bazel_philwo/6d3feea2bf88e88127079b36d7ddade1/execroot). - * Except a user-configurable set of blacklisted files or directories (e.g. /var/secret). - - * Has a separate execroot just for this action: /tmp/_bazel_philwo/6d3feea2bf88e88127079b36d7ddade1/execroot-1 - * which only contains the input files listed for the action. - * to which the output files will be written. - * from which the output files will be moved to the real execroot after successful execution. - - * Processes can write wherever they naturally have permission to do so. - * However, writes have no influence on the host system, instead they are redirected into a separate folder (see: overlayfs's upperdir). - -#### Open issues - - * We don't know a way to hide the workspace, while still making selected input files out of it available inside the new execroot. Ideas we tried: - * copying: Works, but too slow. - * hard-linking: Does not work when workspace and output_base are on different filesystems. - * bind mounting them: Works only on Linux, does not scale (the mount syscall becomes really slow once you're at >20000 active mounts). - * building a custom FUSE filesystem: Might work, but lots of effort. - - * How to hide files from the sandboxed process? - * There seems to be no good way to "hide" files on Linux or macOS. - * The best we can do on both systems is to make them unreadable. - - * overlayfs is not usable for our purposes, because it requires root on all systems except Ubuntu. - -#### What can we do today? - -This is what Bazel is doing for sandboxing at the moment: - - * Allows read access to everything in /. - * Except a [user-configurable](https://github.com/bazelbuild/bazel/blob/0f119a4db515105217244e4db5d4fed9371ef1a4/src/main/java/com/google/devtools/build/lib/sandbox/SandboxOptions.java#L96) set of blacklisted files or directories (e.g. /var/secret). - - * Has a separate execroot just for this action: /tmp/_bazel_philwo/6d3feea2bf88e88127079b36d7ddade1/execroot-1 - * which contains symlinks to the input files listed for the action (the targets are in the workspace or the "real" execroot). - * to which the output files will be written. - * from which the output files will be moved to the real execroot after successful execution. - - * Processes can only write to their private execroot and a private $TMPDIR. - -### Related links - - * [Known issues in this area of work](https://github.com/bazelbuild/bazel/issues?q=is%3Aopen+is%3Aissue+label%3A%22category%3A+sandboxing%22) diff --git a/site/designs/_posts/2016-06-21-environment.md b/site/designs/_posts/2016-06-21-environment.md deleted file mode 100644 index bf1e445b9c..0000000000 --- a/site/designs/_posts/2016-06-21-environment.md +++ /dev/null @@ -1,274 +0,0 @@ ---- -layout: contribute -title: Specifying environment variables ---- - -# Design Document: Specifying environment variables for actions - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Implemented. See [documentation](/docs/skylark/lib/ctx.html#action) - -**Author**: [Klaus Aehlig](mailto:aehlig@google.com) - -**Design document published**: 21 June 2016 - -## Current shortcomings - -Currently, Bazel provides a cleaned set of environment variables to the -actions in order to obtain hermetic builds. This, however is not sufficient -for all use cases. - -* Projects often want to use tools which are not part of the repository; however, - their location varies from installation to installation. So, some sensible - value for the `PATH` environment variable has to be set. - -* Some set-ups depend on every program having access to specific variables, - e.g., indicating the homebrew paths, or library paths. - -* Commercial compilers sometimes need to be passed the location of a license - server through the environment. - -## Proposed solution - -### New flag `--action_env` - -We propose to add a new bazel flag, `--action_env` which has two -valid forms of usage, - -* specifying a variable with unspecified value, `--action_env=VARIABLE`, - and - -* specifying a variable with a value, `--action_env=VARIABLE=VALUE`; - in the latter case, the value can well be the empty string, but it is still - considered a specified value. - -This flag has a "latest wins" semantics in the sense that if the option is given -twice for the same variable, only the latest option will be used, regardless -whether specified or unspecified value. Options given for different variables -accumulate. - -In every action executed -with [`use_default_shell_env`] (/docs/skylark/lib/ctx.html#action) being true, -precisely the environment variables specified by -`--action_env` options are set as the default environment. -(Note that, therefore, by default, the environment for actions is empty.) - -* If the effective option for a variable has an unspecified value, - the value from the invocation environment of Bazel is taken. - -* If the effective option for a variable specifies a value, this value is - taken, regardless of the environment in which Bazel is invoked. - -Environment variables are considered an essential part of an action. In other -words, an action is expected to produce a different output, if the environment -it is invoked in differs; in particular, a previously cached value cannot be -taken if the effective environment changes. - -Given that normally a rule writer cannot know which tools might need fancy -environment variables (think of the commercial compiler use case), the default -for the [`use_default_shell_env`] (/docs/skylark/lib/ctx.html#action) -parameter will become true. - -### List of rc-files read by Bazel - -The list of rc-files that Bazel takes options from will include, at -least, the following files, where files later in the list take precedence over -the ones earlier in the list for conflicting options; for the -`--action_env` option the already described "latest wins" semantics is -applied. - -* A global rc-file. This file typically contains defaults for a whole group of - machines, like all machines of a company. On UNIX-like systems, it will be - located at `/etc/bazel.bazelrc`. - -* A machine-wide rc-file. This file is typically set by the administrator of - the machine or a group of machines with the same architecture. It typically - contains settings that are specific to that architecture and hardware. - On UNIX-like systems it will be next to be binary and called like the binary - with `.bazelrc` appended to the file name. - -* A user-specific file, located in `~/.bazelrc`. This file will be set by - each user for options desired for all Bazel invocations. - -* A project-specific file. This is the file `tools/bazel.rc` next to - the `WORKSPACE` file. This file is considered project-specific and - typically versioned in the same repository as the project. - -* A file specific to user, project, and checkout. This is the file - `.bazelrc` next to the `WORKSPACE` file. As it is specific to - the user and the machine he or she is working on, projects are advised - to ignore that file in the repository of the project (e.g., by adding - it to their `.gitignore` file, if they version the project with git). - -When looking for those rc-files, symbolic links are followed; files not -existing are silently assumed to be empty. Note that all those are regular -rc-files for Bazel, hence are not limited to the newly introduced -`--action_env` option. Also, the rule that options for more specific -invocations win over common options still applies; but, within each level of -specificness, precedence is given according to the mentioned order of rc-files. - -## Example usages of environment specifications - -The proposed solution allows for a variety of use cases, including the -following. - -* Systems using commercial compilers can set the environment variables with - information about the license server in the global rc file. - -* Users requiring special variables, like the ones used by homebrew, can set - them in their machine specific rc-file. In fact, once this proposal is - implemented, the homebrew port for Bazel could itself install that - machine-wide rc-file. - -* Projects depending on the environment, e.g., because they use tools assumed to - be already installed on the user's systm, have several options. - - * If they are optimistic about the environment, e.g., because they are not - very version dependent on the tools used, can just specify which environment - variables they depend on by adding declarations with unspecified values in - the `tools/bazel.rc` file. - - * If dependencies are more delicate, projects can provide a configure script - that does whatever analysis of the environment is necessary and then write - `--action_env` options with specified values to the user-project - local `.bazelrc` - file. As the configure script will only run when manually invoked by the - user and the syntax of the user-project local `.bazelrc` file is so that it - can be easily - be edited by a human, it is OK if that script only works in the majority of - the cases, as a user requiring an unusual setup for that project can easily - modify the user-project local `.bazelrc` by hand afterwards. - -* Irrespectively of the approach chosen by the project, a user where the - environment changes frequently (e.g., on clusters or other machines using a - traditional layout) can fix the environment by adding `--action_env` - options with specific values to the user-project local `.bazelrc`. - - To simplify this use case, and other "freeze on first use" approaches, - Bazel's `info` command will provide a new key `client-env` that will show - the environment variables, together with their values. More precisely, - each variable-value pair will be prefixed with `build --action_env=`, so - that `bazel info client-env >> .bazelrc` can be used to freeze the - environment. - -## Transition plan - -Currently, some users of Bazel already make use of the fact that `PATH`, -`LD_LIBRARY_PATH`, and `TMPDIR` are being passed to actions. To allow those -projects a smooth -transition to the new set up, the global Bazel rc-file provided by upstream -will have the following content. - -``` -build --action_env=PATH -build --action_env=LD_LIBRARY_PATH -build --action_env=TMPDIR -build --test_env=PATH -build --test_env=LD_LIBRARY_PATH -``` - - -## Bazel's own dependency on `PATH` - -Bazel itself also uses external tools, like `cat`, `echo`, `sh`, but also -tools like `bash` where the location differs between installations. In -particular, a value for `PATH` needs to be provided. This will be covered -by the setting of the global bazel configuration file. Should the need arise, a -configure-like script can be added; at the moment it seems that this will not -be necessary. - -## Reasons for the Design Choices, Risks, and Alternatives Considered - -### Conflicting Interests on the environment influencing actions - -There are conflicting requirements for the environment variables of an action. - -* Users expect Bazel to "just work", i.e., the expectation is that if a tool - works on the command line, it should also work when called from an action in - a Bazel invocation from the same environment. A lot of compilers, however, - depend, at least on some systems, on certain environment variables. - An approach used by quite a few other build systems is to pass through the - whole invocation environment. - -* Bazel wants to provide correct and reproducible builds. Therefore, everything - that potentially influences the outcome of an action needs to be controlled - and tracked; a cached result cannot be used if anything potentially changing - the outcome has changed. - -* Users expect Bazel to not do rebuilds they (i.e., the users) know are - unnecessary. And, while for a lot of users the environment variables that - actually influence the build stay stable, the full environment constantly - changes; take the `OLDPWD` environment variable as an example. - -This design tries to reconcile these needs by allowing arbitrary environment -variables being set for actions, but only in an opt-in way. Variables need to -be explicitly mentioned, either in a configuration file or on the command line, -to be provided to an action. - -### Generic Solutions versus Special Casing - -As Bazel already has quite a number of concepts, there is the valid concern -that the complexity might increase too much and newly added concepts might -become a maintenance burden. Another concern is that more configuration -mechanisms make it harder for the user to know which one is the correct one -to use for his or her problem. The general desire is to have few, but powerful -enough mechanisms to control the build behaviour and avoid special casing. - -* Putting the environment variables visible in actions in the hand of the - user avoids the need of special casing more and more "important" environment - variables. - -* Building on the already existing mechanism to specify, inherit, and override - command-line options reduces the amount newly introduced concepts. The main - addition is a command-line option. - -### Source of Knowledge for Needed Environment Variables - -Another aspect that went into the design is that different entities know -about environment variables that are essential for the build to work. - -* Some variables are "obviously" relevant, like `PATH` or `TMPDIR`. - However, there is no "obvious" value for them. - - * Both depend on the layout of the system in question. A special fast - file system for temporary files might be provided at a designated - location. Binaries might be installed under `/bin`, `/usr/bin`, - `/usr/local/bin`, or even versioned paths to allow parallel installations - of different versions of the same tool. For example, on Debian Gnu/Linux - the `bash` is installed in `/bin`, whereas on FreeBSD it is usually - installed in `/usr/local/bin` (but the prefix `/usr/local` is at the - discretion of the system administrator). - - * The user might have custom-built versions of tools somewhere in the - home directory, thus making the user the only one who knows an appropriate - value for the `PATH` variable. Moreover, a user who works on several - projects requiring different versions of the same tool may even require - different values of the `PATH` variable for each project. - -* The authors and users of a tool know about special variables the tools - need to work. While the tool itself might serve a standard purpose, like - compiling C code, the variables the tool depends on might be specific to - that tool (like passing information about a license server). - -* The maintainers of a porting or packaging system know about environment - variables a tool might additionally need (e.g., in the homebrew case). - These might not be needed if the same tool is packaged differently. - -* The project authors know about environment variables special to their - project that some of their actions need. - -These different sources of information make it hard to designate a -single maintainer for the action environment. This makes approaches -undesirable that are based on a single source specifying the action -environment, like the `WORKSPACE` file, or the rule definitions. While -those approaches make it easy to predict the environment an action will -have, they all require the user to merge in the specifics of the system -and his or her personal settings for each checkout (including rebasing -these changes for each upstream change of that file). Collecting environment -variables via the rc-file mechanism allows setting each variable within -the appropriate scope (global, machine-dependent, user-spefic, project-specific, -specific to the user-project pair) in a conflict-free way by the entity -in charge of that scope. diff --git a/site/designs/_posts/2016-08-04-extensibility-for-native-rules.md b/site/designs/_posts/2016-08-04-extensibility-for-native-rules.md deleted file mode 100644 index 1c05d4a5bb..0000000000 --- a/site/designs/_posts/2016-08-04-extensibility-for-native-rules.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -layout: contribute -title: Extensibility For Native Rules ---- - -# Design Document: Extensibility For Native Rules - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Reviewed, not yet implemeted - -**Author**: [Dmitry Lomov](mailto:dslomov@google.com) - -**Design document published**: 04 August 2016 - -## Motivation - -There is a number of requests that require Skylark API to access functionality -of native rules from Skylark rules. Typical scenarios can be illustrated by the -following "sandwich": - -```python -bread_library(name = "top", …) -java_library(name = "meat", deps = [":top", …], …) -bread_library(name = "bottom", deps = [":meat", …]) -``` - -Here bread\_library is a rule written in Skylark. Here we need three things: - -* implementation of bread\_library should be able to produce jar files and other - artifacts in the same way as native java\_library rule would; in other words - the implementation should be able to delegate to the a native implementation -* java\_library should allow depending on bread\_library; importantly, that - dependence should be *meaningful*, that is if bread\_library produces Java - artifacts, such as jar files, java\_library should be able to compile against - those like it would against a java\_library dependency -* bread\_library should be able to depend on java\_library; it should be able to - access all information it needs; when delegating to native implementation, - there should be a simple way to pass information from a dependency to to the - native implementation - -In this document we present some ideas about what how this all might look like, -and suggest some practical steps we can take to get there. - -## Extensible Native Rules - -This proposal assumes [Declared Providers -](/designs/skylark/declared-providers.html) -are implemented. Here is how the implementation of bread\_library might look -like: - -```python -# Implementation of a rule that transpiles to Java and invokes a native -# compilation -def _bread_library_impl(ctx): - bread_sources = [f for src in ctx.attrs.src for f in src.files] - generated_java_files = _invoke_bread_transpiler(ctx, bread_sources) - # lang.java.provider is a declared provider for Java - java_deps = [target[lang.java.provider] for target in ctx.attrs.deps] - # create a native compilation action - java_p = lang.java.compile(ctx, - srcs = generated_java_files, - # information about dependencies is just a lang.java.provider - deps = java_deps, - ...) - # java_p is a lang.java.provider representing the result of compilation - # action we return that provider and immediately java_libary rule can depend - # on us - return [java_p, ...] - -# Implementation of a rule that compiles to JVM bytecode directly -def _scala_library_impl(ctx): - # collect dependency jars to pass to the compile action - dep_jars = [dep[java.lang.provider].jar for dep in ctx.attrs.deps] - jar_output = ctx.new_file(...) - ... construct compilation actions ... - # build a provider that passes all transitive information - transitive_p = lang.java.transitive( - [dep[java.lang.provider] for dep in ctx.attrs.deps]) - java_p = lang.java.provider( - transitive_p, - jar = jar_output, - # update transitive information that we care about - transitive_jars = - transitive_p.transitive_jars | set(jar_output), - ... whatever other information is needed ...) - # return java.lang.provider - return [java_p, ...] -``` - -The provider is the glue ("butter") that connects Skylark rules to native rules -and also to the native rule implementations exposed to Skylark. Note how the -native rule implementation (lang.java.compile) both consumes the entire -providers from dependencies and returns the provider that needs to be returned -from the rule. `lang.java.transitive` is a function that passes all the -transitive information correctly from dependencies. The [existing '.java' -provider](http://www.bazel.build/docs/skylark/lib/JavaSkylarkApiProvider.html) -becomes the same thing as lang.java.provider. - -Note: for the sake for this document we are placing things in lang.java. There -are other alternatives to this, e.g. "magical" .bzl files from which -java\_provider and java\_compile function are exported. - -## How to get there - -Our current native rules are not as neat as described above. Making them -extensible in one go is a difficult and long term project (or rather, projects: -one for each language). Here is a suggested steps for extensibility of -particular language implementation (we continue to use Java as a running -example). - -### Phase 1: Expose native compilation actions - -At this step, lang.java.provider is a *black box*. Skylark rules cannot -construct the lang.java.provider directly: the only way to create it is to -invoke lang.java.compile function. - -Native implementations of Java rules are rewritten so that they can link to deps -that return lang.java.provider and that they return lang.java.provider. The -implementation of the provider can just be a bag of all providers that Java -rules normally return - since that bag is not openable by Skylark, we can -refactor it later without much difficulty . - -Native compilation function (java.lang.compile) is pretty much -JavaLibrary.create refactored so that it gets its dependent providers not from -attributes but from a list of bags. JavaLibrary.create just collects the bags -from deps and passes those to that function. - -At the end of this phase, implementing code generators (and code generating -aspects) such as java\_proto\_library becomes possible. This also covers many -(most?) use cases where people use macros to delegate to native rule -implementations - -No huge refactoring of language rule implementation is needed, but the stage is -set for gradual opening up in the future. - -### Phase 1a: Implementing JavaSkylarkApiProvider on top of black box - -(Optional) As lang.java.provider is just a bag of existing providers, it is easy -to just implement everything in 'target.java' on top of it, if desired. - -### Phase 2: Evolving the API and opening up - -The next step in the API evolution is making the black box provider less black. -This means introducing a constructor for lang.java.provider as well as accessors -to fields. - -The API can be designed gradually and thoughtfully, only exposing the things we -need and adding carefully: as an example sequence first just java libraries, -then resources, then JNI, then support for tests. Existence of -lang.java.transitive is crucial at this stage as it allows merging of transitive -information from dependencies that is not yet exposed to Skylark. - -As API exposure gradually progresses, the exposed Skylark API reaches parity -with internal API. - -Through the execution of this phase, more and more use cases are covered, and at -the end the rules are fully extensible. diff --git a/site/designs/_posts/2016-09-05-build-python-on-windows.md b/site/designs/_posts/2016-09-05-build-python-on-windows.md deleted file mode 100644 index 312575c89e..0000000000 --- a/site/designs/_posts/2016-09-05-build-python-on-windows.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -layout: contribute -title: Building Python on Windows ---- - -# Design Document: Building Python on Windows - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Implemented - -**Author**: [Yun Peng](mailto:pcloudy@google.com) - -**Design document published**: 05 September 2016 - -**Relevant changes**: - -1. [zipper now can specify actual path a file is added to](https://bazel-review.googlesource.com/4243) -2. [Create Python executable zip file](https://bazel-review.googlesource.com/4244) -3. [Make python executable zip a real self-extracting binary](https://bazel-review.googlesource.com/4263) -4. [Using stub\_template.txt as \_\_main\_\_.py and zip header in python executable zip](https://bazel-review.googlesource.com/5310) -5. [Get rid of python executable zip file header](https://bazel-review.googlesource.com/5350) -6. [Put runfiles tree under 'runfiles' directory to avoid conflict](https://bazel-review.googlesource.com/5351) - -As we keep finding new problems and coming up with new solutions, you -can see some of the implementation in the previous changes has been -deprecated in the latter ones. Here we only present the final solution. - -## Motivation - -After providing basic support for C++ and Java on Windows (although not -perfect for C++ due to the wrapper scripts we use), Python becomes the -last language we need to fix in order to make Bazel no longer -experimental on Windows. - -## Problem - -Currently, as described on [bazel.build](/docs/be/python.html#py_binary), -the way py\_binary works on Unix is: - -_A py\_binary is an executable Python program consisting of a collection -of `.py` source files (possibly belonging to other py\_library rules), -a `*.runfiles` directory tree containing all the code and data needed by -the program at run-time, and a [stub script](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/bazel/rules/python/stub_template.txt) -that starts up the program with the correct initial environment and data._ - -This doesn’t work on Windows, because we don’t have runfiles tree on Windows. -A runfiles client library (which maps a runfile from runfiles path to -its real path) doesn’t solve the whole problem, because we also need -`__init__.py` file under every python source directory to make the -directory a recognizable python package. And it doesn’t make sense to -create `__init__.py` files outside of runfiles tree. Therefore, how to -find a way to run python binary properly on Windows is our main problem. - -## Solution - -Python has the ability to execute zip file as scripts since version 2.6. -When invoking the python interpreter with a zip file as the first argument, -it executes the `__main__.py` file in the root directory of the archive. - -The idea of the solution is to create a self-extracting zip file which -packages everything supposed to be in the original runfiles tree and add -the stub script as the `__main__.py` file. We tell the stub script whether -it’s in a zip file or not. If it is, it first extracts the zip file to a -temporary directory as the runfiles tree, then set the correct environment -variables and runs the main python script. At the end of the execution, -it deletes the temporary directory. - -We did three more things to achieve this goal: - -#### 1. Implement new feature in zipper for packaging runfiles tree easily - -Creating the zip file is not a trivial thing, since we don’t have runfiles -tree at all. We should not only archive every runfile into the right path, -but also adding `__init__.py` file to every directory. To make things -easier, we introduce a new feature in zipper (a custom zip tool of Bazel) -which makes users able to specify the actual path a file is archived into. -Zipper now supports the following semantics: - -`zipper cC x.zip [<zip_path>=][<file>]` - -Examples: - -```bash -$ zipper cC x.zip a/b/lib.py # Add file a/b/lib.py -$ zipper cC x.zip a/b/__init__.py= # Add an empty file at a/b/__init__.py -$ zipper cC x.zip a/b/main.py=foo/bar/bin.py # Add file foo/bar/bin.py as a/b/main.py -``` - -With the help of this feature, we can easily control the directory -structure in the zip file. - -#### 2. Build the final python binary - -The final python binary is the zip file with a `#!/usr/bin/env python` -shebang. By doing this, we can run the binary in the following two ways: - -```bash -$ ./bazel-bin/foo/bar/bin -$ python ./bazel-bin/foo/bar/bin -``` - -And we can use the second way to run the binary from Windows native -command line(cmd.exe). Since the zip file packages everything in runfiles -tree, it can be copied to anywhere as a self-contained executable binary. - -#### 3. Add `--build_python_zip` flag - -This flag can be used to tell Bazel whether or not it should build a zip file -as the final binary. By default, it’s enabled on Windows, -disabled on other platform. diff --git a/site/designs/_posts/2016-09-19-recursive-ws-parsing.md b/site/designs/_posts/2016-09-19-recursive-ws-parsing.md deleted file mode 100644 index 9261d3b3f5..0000000000 --- a/site/designs/_posts/2016-09-19-recursive-ws-parsing.md +++ /dev/null @@ -1,206 +0,0 @@ ---- -layout: contribute -title: Recursive WORKSPACE file parsing ---- - -# Design Document: Recursive WORKSPACE file parsing - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Unimplemented - -**Author**: [kchodorow@](mailto:kchodorow@google.com) - -**Design document published**: 19 September 2016 - -## Objective - -Users are annoyed by having to specify all deps in their WORKSPACE file. To -avoid this inconvenience, Bazel could load subprojects' WORKSPACE files -automatically. - -## Non-Goals - -* Solve the problem of people specifying two different repositories with the - same name (e.g., @util). -* Solve the problem of people specifying two different names for the same - repository (`@guava` and `@com_google_guava`). - -## Resolution - -When a repository is defined in multiple files, which definition "wins"? What -causes a conflict/error? - -### Defined in the main repository's WORKSPACE file - -This definition wins, regardless of other definitions. - -### In a line - -Note: as an intermediate step, this can be disabled, but the end goal is to -allow this so that intermediate dependencies that the top level doesn't care -about don't need to be resolved. - -Suppose we have a main repository that depends on repo x, and x depends on repo -y: - -<img src="/assets/ws-line.png" class="img-responsive"> - -In this case, version 1 of "foo" wins. This way, if a library has already -figured out which version works for them, its reverse dependencies do not have -to think about it. - -This will also work if a parent overrides its children's versions, even if it -has multiple children. - -### Different lines - -If there is no obvious hierarchy and multiple versions are specified, error out. -Report what each chain of dependencies was that wanted the dep and at which -versions: - -<img src="/assets/ws-multiline.png" class="img-responsive"> - -In this case, Bazel would error out with: - -``` -ERROR: Conflicting definitions of 'foo': bazel-external/y/WORKSPACE:2 repository(name = 'foo' version = '1') - requested by bazel-external/x/WORKSPACE:2 repository(name = 'y') - requested by WORKSPACE:3 repository(name = 'x') -vs. bazel-external/a/WORKSPACE:2 repository(name = 'foo' version = '2') - requested by WORKSPACE:2 repository(name = 'a') -``` - -This is also the case with diamond dependencies: - -<img src="/assets/ws-diamond.png" class="img-responsive"> - -This would print: - -``` -ERROR: Conflicting definitions of 'foo': bazel-external/x/WORKSPACE:2 repository(name = 'foo' version = '2') - requested by WORKSPACE:2 repository(name = 'x') -vs. bazel-external/z/WORKSPACE:2 repository(name = 'foo' version = '1') - requested by bazel-external/y/WORKSPACE:2 repository(name = 'z') - requested by WORKSPACE:3 repository(name = 'y') -``` - -## Upgrade path - -I think that this should be fairly straightforward, as any repository used by -the main repository or any subrepository had to be declared in the WORKSPACE -already, so it will take precedence. - -To be extra safe, we can start with adding a `recursive = False` attribute to -the `workspace` rule, which we can then flip the default of. - -## Implementation - -There are two options for implementing this: - -* We always download/link every external dependency before a build can happen. - E.g., if @x isn't defined in the WORKSPACE file, we have to recursively - traverse all of the repositories to know which repository does define it and - if there are any conflicting definitions. This is correct, but will be - frustrating to users and may not even work in some cases (e.g., if an - OS-X-only skylark repository rule is fetched on Linux). -* Every time a new WORKSPACE file is fetched, we check its repository rules - against the ones already defined and look for version conflicts. This would - entirely miss certain version conflicts until certain dependencies are built, - but will have better performance. - -I think users will rebel unless we go with Option 2. However, this can have -some weird effects: suppose we have the diamond dependency above, and the user's -BUILD file contains: - -``` -cc_library( - name = "bar", - deps = ["@x//:dep"], # using @foo version 2 -) - -cc_library( - name = "baz", - deps = ["@y//:dep"], # using @foo version 1 -) -``` - -If they build :bar and their coworker builds :baz, the two builds will work and -get different versions of @foo. However, as soon as one of them tries to build -both, they'll get the version mismatch error. - -This is suboptimal, but I can't think of a way that all three of these can be -satisfied: - -* The user doesn't have to declare everything at the top level. -* Bazel doesn't have to load everything. -* Bazel can immediately detect any conflicts. - -This could be enforced by a CI on presubmit, which I think is good enough. - -Whenever Bazel creates a new repository, it will attempt to parse the WORKSPACE -file and do Skyframe lookups against each repository name. If the repository -name is not defined, it will be initialized to the current WORKSPACE's -definition. If it already exists, the existing value will be compared. - -For now, we'll be very picky about equality: `maven_jar` and `new_http_archive` -of the same Maven artifact will count as different repositories. For both -native and skylark repository rules, they will have to be equal to not conflict. - -One issue is that is a little tricky but I think will work out: the WORKSPACE -file is parsed incrementally. Suppose the main WORKSPACE loads x.bzl, which -declares @y and @z. If @y depends on @foo version 1 and @z depends on @foo -version 2, this will throw a Skyframe error, even if @foo is later declared in -the WORKSPACE file. However, this should be okay, because if these dependencies -actually need @foo, it would need to be declared before them in the WS file -already. - -## Supplementary changes - -Not strictly required, but as part of this I'm planning to implement: - -* A `bazel-external` convenience symlink (to the `[output_base]/external` - directory) so users can easily inspect their external repositories. -* Add an option to generate all WORKSPACE definitions (so generate a flat - WORKSPACE file from the hierarchy). - -## Concerns - -Questions users might have. - -*Where did @x come from?* - -Bazel will create a `bazel-external/@x.version` should contain the WORKSPACE (or -.bzl file) where we got @x's def and other WORKSPACE files that contain it. - -*Which version of @x is going to be chosen?* - -See resolution section above. Perhaps people could query for //external:x? - -*I want to use a different version of @x.* - -Declare @x in your WORKSPACE file, it'll override "lower" rules. - -*When I update @x, what else will change?* - -Because @x might declare repo @y and @y's version might change as well, we'd -need a different way to query for this. We could implement deps() for repo -rules or have some other mechanism for this. - -## Thoughts on future development - -Moving towards the user-as-conflict-resolver model (vs. the -user-as-transcriber-of-deps model) means that repositories that the user may not -even be aware of might be available in their workspace. I think this kind of -paves the way towards a nice auto-fetch system where a user could just depend on -`@com_google_guava//whatever` in their BUILD file, and Bazel could figure out -how to make `@com_google_guava` available. - -## References - -[So you want to write a package manager](https://medium.com/@sdboyer/so-you-want-to-write-a-package-manager-4ae9c17d9527#.d90oxolzk) -does a good job outlining many of these challenges, but their suggested approach -(use semantic versioning for dependency resolution) cannot be used by Bazel for -the general case. diff --git a/site/designs/_posts/2016-10-11-distribution-artifact.md b/site/designs/_posts/2016-10-11-distribution-artifact.md deleted file mode 100644 index a1bbd9acca..0000000000 --- a/site/designs/_posts/2016-10-11-distribution-artifact.md +++ /dev/null @@ -1,143 +0,0 @@ ---- -layout: contribute -title: Distribution Artifact for Bazel ---- - -# Design Document: Distribution Artifact for Bazel - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Implemented - -**Author**: [Klaus Aehlig](mailto:aehlig@google.com) - -**Design document published**: 11 October 2016 - -## Current State and Shortcomings - - -### Dependency on `protoc` - -Bazel depends on a protobuffer compiler to generate code, especially -java code, from an abstract description of the protocol buffer; -in particular, files generated by `protoc` are machine-independent. -In fact, Bazel most of the time uses the latest version of `protoc`. -New versions of `protoc` that contain incompatible changes to the -programming interface are released frequently. - -### Current approach to this dependency - -The current approach to the `protoc` dependency is to have checked-in -statically-linked executables for all the supported platforms (where -some platforms, like FreeBSD, have to use Linux-compatibility features). -The full source tree of the protobuf compiler is also part of the repository. -However, for generating files, the committed binaries are always used. - -### Shortcomings - -The current approach as certain shortcomings. - -- Having up-to-date binaries for all the supported platforms does not scale well - as the number of platforms Bazel should run on is increasing. - -- The requirement of having a suitable executable in the code base adds - additional complexity to the process of bootstrapping a new architecture. - -- Binaries in the code base do not follow standard open-source principles; in - fact, meaningful reviews for changes updating them are hard and in practise - often boil down to a question of trust in the person making the change. - -- Committed binaries make the "source" repository unnecessary big. Currently, - a checkout at head contains over 250MB in committed `.exe` and `.dll` files. - -## Proposed solution - -### Change `BUILD` to compile `protoc` from source - -This `BUILD` file for the `third_party/protobuf` is changed in such a -way, that the `protoc` is compiled from source instead of selecting from -the committed pre-built binaries; the pre-built binaries are removed from -the source tree. As the `protoc` sources are already part of the repository, -this is not a huge change; also, as `protoc` is written in `C++`, no additional -dependencies are introduced that way. - -Note that then, every user who already has a working (bootstrap) `bazel`, can -build bazel from source, without depending on committed binaries or having -a `protoc` already on the machine. The problem of building your first `bazel` -will be addressed in the next sections. - -This change also removes an internal consistency requirement from the code -base. It was always assumed that the binaries actually match the accompanying -sources. - -### Distribution artifact - -A new target `//:bazel-distfile` will be added. This will be an archive -containing - -- all source files in their respective places, including the files - under `third_party`, `site`, `scripts`, etc, as well as - -- under a subdirectory `derived` all the files generated by `protoc` that - are needed to compile a bootstrap version of `bazel`. - -For convenience, the `derived` subdirectory may also contain other -generated architecture-independent files, like an HTML-version of the -documentation for local browsing. A corrollary of the archive layout is that -by removing the `derived` directory a checkout of the upstream sources is -obtained. - -This new artifact will be built for every release and made available together -along with the other release artifacts (like packages, installers, executables). -The same means of certifying integrity (like hashes, SSL-certificates) will be -used. - -### Bootstrapping Bazel - -The `compile.sh` will be modified to first check if a `derived` directory exists -and if this is the case assume that all the files generated by `protoc` are -already present there; only if not present, it will try to generate the needed -output of `protoc` for bootstrapping, assuming that the `PROTOC` environment -variable points to a good `protoc` binary. - -So, there will be three ways to build `bazel`. - -- If one has an old `bazel` binary already, a new one can be built from a - checkout of the source repository. This approach is useful for developpers. - It might also be used by users who want to upgrade their old `bazel` binary - to the next release. - -- By downloading the distribution artifact, the `compile.sh` script can be - used to build bazel. Again, no `protoc` has to be installed ahead of time. - This approach is useful for source distributions, as well as for bringing - Bazel to a new platform. - -- If one already has the correct version of `protoc` on the machine, the - `compile.sh` script can be used by setting the `PROTOC` environment variable. - This approach is useful for distributions that want to provide snapshots - of `bazel` inbetween official releases and maintain a `protoc` package anyway. - -## Other approaches considered - -### Requiring users to have the correct version of the `protoc` binary installed - -This would be the standard open-source approach of requiring the user to have -the required dependencies installed ahead of time. Unfortunately, `protoc` -contains incompatible changes too frequently, so that this would be an -unreasonable -burden. Note that the bootstrapping from your own `protoc` and a repository -checkout is still possible with the suggested approach. - -### Committing the `protoc` output - -Another approach would be to make the output of `protoc` part of the versioned -sources instead of generating them for the distribution file. As with all -approaches based on committing generated files, this would -introduce another consistency requirement to the repository. In this case, the -requirement would be that the generated files be up-to-date with respect to the -respective `.proto` files. Of course, such a consistency could be verified by -an appropriate test. Nevertheless, it seems the cleaner and probably more -managable to only version true source files and generate derived files from -the respective sources. diff --git a/site/designs/_posts/2016-10-18-repository-invalidation.md b/site/designs/_posts/2016-10-18-repository-invalidation.md deleted file mode 100644 index fc0f8fec14..0000000000 --- a/site/designs/_posts/2016-10-18-repository-invalidation.md +++ /dev/null @@ -1,163 +0,0 @@ ---- -layout: contribute -title: Invalidation of remote repositories ---- - -# Design Document: Invalidation of remote repositories - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Implemented - -**Author**: [Damien Martin-Guillerez](dmarting@google.com) - -**Design document published**: 18 October 2016 - -## State at commit [808a651](https://github.com/bazelbuild/bazel/commit/808a6518519501cfd32755a229d5dddf70e33557) - -[Remote repositories](/docs/external.html) are fetched the first -time a build that depends on a repository is launched. The next -time the same build happens, the already fetched repositories -are not refetched, saving on download times or other expensive -operations. - -This behavior is also enforced even when the Bazel server -is restarted by serializing the repository rule in the workspace -file. A file named `@<repositoryName>.marker` is -[created](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java#L131) -for each repository with a -[fingerprint of the serialized rule](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java#L192). On -next fetch, if that fingerprint has not changed, the rule is not -refetched. This is not applied if the repository rule is marked -as -[`local`](https://www.bazel.io/versions/master/docs/skylark/lib/globals.html#repository_rule) -because fetching a local repository is assumed to be -[fast](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorFunction.java#L125). - -## Shortcomings - -These consideration were well-suited when the implementation of -repository rules were not depending on Skylark file. With the introduction of -[Skylark repositories](https://www.bazel.io/versions/master/docs/skylark/repository_rules.html), -several issues appeared: - -- [Change in the skylark implementation of the rule does not - trigger a refetch of the rule](https://github.com/bazelbuild/bazel/issues/1022), - nor does a change in one of the template files that relies on that - rule: the rule marker does not contains this information. -- There is [no way to re-configure a repository used for - auto-configuration](https://github.com/bazelbuild/bazel/issues/974), - leading to - [excessive uses of `bazel clean --expunge`](https://github.com/tensorflow/tensorflow/blob/60d54d6b8524bcaf512f53384b307fae47b953d2/configure#L25). -- The invalidation behavior of repository rules are unclear and - difficult to explain. - -## Proposed solution - -### Invalidation on the environment - -Right now rules are not invalidated on the environment: - -- Invalidation on accessing - [`repository_ctx.os.environ`](https://www.bazel.io/versions/master/docs/skylark/lib/repository_os.html#environ) - would generate invalidation on environment variable that might be - volatile (e.g. `CC` when you want to use one C++ compiler and you - reset your environment) and might miss other environment variables - due to computed variable names. -- There is no way to represent environment variables that influence - [`repository_ctx.execute`](https://www.bazel.io/versions/master/docs/skylark/lib/repository_ctx.html#execute). - -This document proposes to add a way to declare a dependency on an -environment variable value that would trigger a refetch of a -repository. An optional attribute `environ` would be added to the -[`repository_rule`](https://www.bazel.io/versions/master/docs/skylark/lib/globals.html#repository_rule) -method, taking a list of strings and would trigger invalidation of the -repository on any of change to those environment variables. E.g.: - -```python -my_repo = repository_rule(impl = _impl, environ = ["FOO", "BAR"]) -``` - -`my_repo` would be refetched on any change to the environment -variables `FOO` or `BAR` but not if the environment variable `BAZ` -would changes. - -To be consistent with the -[new environment specification](https://www.bazel.io/designs/2016/06/21/environment.html) -mechanism, the environment available through -[`repository_ctx.os.environ`](https://www.bazel.io/versions/master/docs/skylark/lib/repository_os.html#environ) -or transmitted to -[`repository_ctx.execute`](https://www.bazel.io/versions/master/docs/skylark/lib/repository_ctx.html#execute) -will take values from the `--action_env` flag, when specified. I.e. if -`--action_env FOO=BAR --action_env BAR` are specified, and the -environment set `FOO=BAZ`, `BAR=FOO`, `BAZ=BAR`, then the actual -`repository_ctx.os.environ` map would contain `{"FOO": "BAR", "BAR": -"FOO", "BAZ": "BAR" }`. This would ensure that the environment seen by -repository rules is consistent with the one seen by actions (a -repository rule see more than an action, leaving the rule -writer the ability to filter the environment more finely). - -Both these changes should allow Bazel to do auto-configuration -based on environment variables: - -- Setting some environment variables would actually retrigger - auto-configuration, corresponding to how the rule writter designed - it (and not based on some assumption from Bazel). -- The user set specific environment variables through the `--action_env` - flag, and fix this environment using `bazel info client-env`. - -### Serialization of Skyframe dependencies - -A `local` rule will be invalidated when any of its skyframe -dependencies change. For non-`local` rule, a marker file -will be stored on the external directory with a summary of the -dependencies of the rule. At each fetch operation, we check -the existence of the marker file and verify each dependency. -If one of them have changed, we would refetch that repository. - -To avoid unnecessary re-download of artifacts, a content-addressable -cache has been developed for downloads (and thus not discuted here). - -The marker file will be a manifest containing the following -items: - -- A fingerprint of the serialized rule and the rule specific data - (e.g., maven server information for `maven_jar`). -- The declared environment (list of name, value pairs) through the - `environ` attribute of the repository rule. -- The list of `FileValue`-s requested by - [`getPathFromLabel`](https://github.com/bazelbuild/bazel/blob/master/src/main/java/com/google/devtools/build/lib/bazel/repository/skylark/SkylarkRepositoryContext.java#L613) - and the corresponding file content digest. -- The transtive hash of the `Extension` definining the repository rule. - This transitive hash is computed from the hash of the current extension - and the extension loaded from it. This means that a repository function - will get invalidated as soon as the extension file content changes, which - is an over invalidation. However, getting an optimal result would require - correct serialization of Skylark extensions. - -## Implementation plan - -1. Modify the `SkylarkRepositoryFunction#getClientEnvironment` method - to get the values from the `--action_env` flag. -2. Adds a `markerData` map argument to `RepositoryFunction#fetch` so - `SkylarkRepositoryFunction` can include those change. This attribute - should be mutable so a repository can add more data to be stored - in the marker file. Adds a corresponding function for - verification, `verifyMarkerManifest`, that would take a marker data - map and return a tri-state: true if the repository is up to date, - false if it needs refetch and null if additional Skyframe dependency - need to be resolved for answering. -3. Add the `environ` attribute to the `repository_rule` function and - the dependency on the Skyframe values for the environment. Also create - a `SkyFunction` for processed environment after the `--action_env` - flag. -4. Adds the `environ` values to the marker file through the - `getMarkerManifest` function. -5. Adds the `FileValue`-s to the marker file, adding all the files - requested through the `getPath` method to a specific builder that - will be passed to the `SkylarkRepositoryContext`. -6. Adds the extension to the marker file by passing the - `transitiveHashCode` of the Skylark `Environment` to the marker - manifest. diff --git a/site/designs/_posts/2017-03-03-label-keyed-string-dict-type.md b/site/designs/_posts/2017-03-03-label-keyed-string-dict-type.md deleted file mode 100644 index 4e0adbdc89..0000000000 --- a/site/designs/_posts/2017-03-03-label-keyed-string-dict-type.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -layout: contribute -title: Label-keyed String Dictionary Type for Build Attributes ---- - -# Design Document: Label-keyed String Dictionary Type for Build Attributes - -**Design documents are not descriptions of the current functionality of Bazel. -Always go to the documentation for current information.** - - -**Status**: Implemented in -[5e9e194](https://github.com/bazelbuild/bazel/commit/5e9e1949f4c08ce09665b92aadf7ec7e518aab6a) - -**Author**: [Michael Staib](mstaib@google.com) - -**Design document published**: 03 March 2017 - -## Background/Motivation - -For future work in the realm of allowing Bazel users to define configuration -flags, the `config_setting` rule will need to be able to test configuration -values defined by labels rather than strings. The current solution uses a -dictionary from string (the flag to check the value of) to string (the value -to check against). It makes sense, then, to use a dictionary from label (the -flag to check the value of) to string (the value to check against) for -user-defined configuration which is defined by a label. - -Additionally, for work which relates to setting such user-defined configuration, -rules should be able to declare similar dictionaries for the purposes of setting -those same flags. - -An example incorporating both testing and setting such flags: - -``` -flag_rule( - name = "beep", - values = ["boop", "bop", "bump"], - default = "bump" -) - -config_setting( - name = "beep#boop", - flag_values = { - ":beep": "boop" - } -) - -transition_rule( - name = "configuration", - deps = [ - ":lib" - ], - sets_flags = { - ":beep": "boop" - } -) - -library_rule( - name = "lib" - deps = select({ - ":beep#boop": [":boop_dep"], - "//conditions:default": [":other_dep"] - }) -) -``` - -## New attribute type: LABEL_KEYED_STRING_DICT -In order to handle these flag values, the BUILD language will need the ability -to express a mapping from a label (a flag's label, to be precise) to a string -(the flag's value). This will be added as `BuildType.LABEL_KEYED_STRING_DICT` in -native rules, and as `attr.label_keyed_string_dict()` in Skylark (taking the -same parameters as `attr.label_list()`). This will have to be serializable -to query `--output=proto` format. - -### Native rule representation -Native rules will be able to take the attribute's value using an -`AttributeMapper`, as normal. In this case, the type returned will be -`Map<Label, String>`. In conjunction with `RuleContext.getPrerequisites`, this -can be used to get both the target and the string value associated with it by -iterating over the return value from `getPrerequisites` and looking up the -labels of the `TransitiveInfoCollection`s in the map. - -### Skylark representation -Skylark rules must render some representation of this structure in -`ctx.attr.<attrname>`. The only restriction on Skylark dictionary keys is that -they must be immutable, which the various `ConfiguredTarget`s are (although they -must be annotated as such). Accordingly, the value of `ctx.attr.<attrname>` is a -dictionary mapping Target to string. This will have to be changed to be another -special case in the `SkylarkRuleContext`. - -Because each target in an attribute will undergo the same transition - if any - -and the transition of the target itself will always be the same, the keys of -this dict will be unique - i.e., there will be no collisions - as long as the -labels used to construct it were unique. - -### Handling collisions when converting attribute values from Skylark -Labels are special in that there are multiple ways (and possibly multiple -encodings!) to represent them in a BUILD or Skylark file which are not the same -from Skylark's point of view. In the package `//label`, the strings `"label"`, -`":label"`, `"//label"`, and `"//label:label"` all evaluate to the same `Label` -when they are picked up by Bazel, but they will be different keys in the dict -created by Skylark, where they are merely strings. Skylark does have a label -type (constructed with `Label("//label")`, yet another way of representing the -same label), and Bazel does accept it for `LABEL` attributes, but most uses of -label-type attributes take advantage of Bazel's automatic conversion of strings -in label-type attributes. That conversion does not happen until the Skylark -value enters the build system at a rule attribute, at which point the value may -have been mutated, read, and passed around in Skylark several times. - -In Skylark, it is an error for a dictionary literal to contain multiple items -with the same key. For consistency and simplicity, `LABEL_KEYED_STRING_DICT` -will throw a `ConversionException` in its convert method if two of the Skylark -dict's keys evaluate to the same label, even if they also have the same value. -This only covers the case where the two keys are distinct strings; if two -identical keys are used in a dictionary literal, there will be an error in -Skylark before this logic ever sees it. Mutations of a key (i.e., -`dictionary[key] = value` for a `key` which is already in the dictionary) will -continue to be allowed as normal. - -## Testing Plan - -* Conversion exception for non-dict values -* Conversion exception for dicts other than string-to-string -* Conversion exception for dicts with multiple keys evaluating to the same label -* Conversion exception for dicts with invalid labels as keys -* Successfully converts to `Map<Label, String>` -* Successfully converts to query proto -* Successfully converts to query XML -* Successfully outputs in build format from query -* visitLabels visits the labels in the keys -* Skylark can define `label_keyed_string_dict` attributes and receive them as a - dict of Target to string -* Skylark can define `label_keyed_string_dict` attributes with provider - requirements and have them be respected -* Skylark can define `label_keyed_string_dict` attributes with filetype - requirements and have them be respected -* Skylark can define `label_keyed_string_dict` attributes and require they not - be empty -* Skylark can define `label_keyed_string_dict` attributes and make them - mandatory -* Skylark can define `label_keyed_string_dict` attributes and set the default - value -* Skylark can define `label_keyed_string_dict` attributes and have Aspects - follow them diff --git a/site/designs/index.md b/site/designs/index.md deleted file mode 100644 index 6792ee2449..0000000000 --- a/site/designs/index.md +++ /dev/null @@ -1,44 +0,0 @@ ---- -layout: contribute -title: Design Documents ---- - -# Design Documents - -The design documents published here are part of a design process: - -1. Design is reviewed -2. Design document is published here -3. Design is implemented - -It's possible that designs change as they are implemented in practice. The -design documents published here capture the initial design, and _not_ the -ongoing changes as designs are implemented. - -Always go to the [documentation](/versions/master/docs/bazel-overview.md) for -descriptions of current Bazel functionality. - -## Bazel Code Style and Best Practices - -* [Implementing Native Declared Provders](https://docs.google.com/document/d/1BKCBuYvf6fwmMnFu3KMWdRFysDgGCzdMiNxY_foosAM/preview) 12 April 2017 - -## Bazel Design Documents - -<ul> -{% for doc in site.categories.designs %} - <li><a href="{{ doc.url }}">{{ doc.title }}</a> - {{ doc.date | date_to_long_string }}</a></li> -{% endfor %} -</ul> - - - -## Skylark Design Documents - -Changes to the Bazel build and extension language (Skylark) should go -through the [Skylark Design Process](/designs/skylark/skylark-design-process.html). - -* [Parameterized Skylark Aspects](/designs/skylark/parameterized-aspects.html) - 18 April 2016 -* [Declared Providers](/designs/skylark/declared-providers.html) 06 June 2016 -* [Saner Skylark Sets](/designs/skylark/saner-skylark-sets.html) 25 July 2016 diff --git a/site/designs/skyframe.md b/site/designs/skyframe.md deleted file mode 100644 index ae3dc53aee..0000000000 --- a/site/designs/skyframe.md +++ /dev/null @@ -1,189 +0,0 @@ ---- -layout: contribute -title: Skyframe ---- - -# Skyframe - -<p class="lead">The parallel evaluation and incrementality model of Bazel</p> - -## Data model - -The data model consists of the following items: - - - `SkyValue`. Also called nodes. `SkyValues` are immutable objects that contain all the data built - over the course of the build and the inputs of the build. Examples are: input files, output - files, targets and configured targets. - - `SkyKey`. A short immutable name to reference a `SkyValue`, for example, `FILECONTENTS:/tmp/foo` - or `PACKAGE://foo`. - - `SkyFunction`. Builds nodes based on their keys and dependent nodes. - - Node graph. A data structure containing the dependency relationship between nodes. - - `Skyframe`. Code name for the incremental evaluation framework Bazel is based on. - - -## Evaluation - -A build consists of evaluating the node that represents the build request (this is the state we are -striving for, but there is a lot of legacy code in the way). First its `SkyFunction` is found and -called with the key of the top-level `SkyKey`. The function then requests the evaluation of the -nodes it needs to evaluate the top-level node, which in turn result in other function invocations, -and so on, until the leaf nodes are reached (which are usually nodes representing input files in -the file system). Finally, we end up with the value of the top-level `SkyValue`, some side effects -(e.g. output files in the file system) and a directed acyclic graph of the dependencies between the -nodes that were involved in the build. - -A `SkyFunction` can request `SkyKeys` in multiple passes if it cannot tell in advance all of the -nodes it needs to do its job. A simple example is evaluating an input file node that turns out to -be a symlink: the function tries to read the file, realizes that it's a symlink, and thus fetches -the file system node representing the target of the symlink. But that itself can be a symlink, in -which case the original function will need to fetch its target, too. - -The functions are represented in the code by the interface `SkyFunction` and the services -provided to it by an interface called `SkyFunction.Environment`. These are the things functions can -do: - - - Request the evaluation of another node by way of calling `env.getValue`. If the node is - available, its value is returned, otherwise, `null` is returned and the function itself is - expected to return `null`. In the latter case, the dependent node is evaluated, and then the - original node builder is invoked again, but this time the same `env.getValue` call will return a - non-`null` value. - - Request the evaluation of multiple other nodes by calling `env.getValues()`. This does - essentially the same, except that the dependent nodes are evaluated in parallel. - - Do computation during their invocation - - Have side effects, for example, writing files to the file system. Care needs to be taken that two - different functions do not step on each other's toes. In general, write side effects (where - data flows outwards from Bazel) are okay, read side effects (where data flows inwards into Bazel - without a registered dependency) are not, because they are an unregistered dependency and as - such, can cause incorrect incremental builds. - -`SkyFunction` implementations should not access data in any other way than requesting dependencies -(e.g. by directly reading the file system), because that results in Bazel not registering the data -dependency on the file that was read, thus resulting in incorrect incremental builds. - -Once a function has enough data to do its job, it should return a non-`null` value indicating -completion. - -This evaluation strategy has a number of benefits: - - - Hermeticity. If functions only request input data by way of depending on other nodes, Bazel - can guarantee that if the input state is the same, the same data is returned. If all sky - functions are deterministic, this means that the whole build will also be deterministic. - - Correct and perfect incrementality. If all the input data of all functions is recorded, Bazel - can invalidate only the exact set of nodes that need to be invalidated when the input data - changes. - - Parallelism. Since functions can only interact with each other by way of requesting - dependencies, functions that do not depend on each other can be run in parallel and Bazel can - guarantee that the result is the same as if they were run sequentially. - -## Incrementality - -Since functions can only access input data by depending on other nodes, Bazel can build up a -complete data flow graph from the input files to the output files, and use this information to only -rebuild those nodes that actually need to be rebuilt: the reverse transitive closure of the set of -changed input files. - -In particular, two possible incrementality strategies exist: the bottom-up one and the top-down one. -Which one is optimal depends on how the dependency graph looks like. - - - During bottom-up invalidation, after a graph is built and the set of changed inputs is known, - all the nodes are invalidated that transitively depend on changed files. This is optimal - if we know that the same top-level node will be built again. - Note that bottom-up invalidation requires running `stat()` on all input files of the previous - build to determine if they were changed. This can be improved by using `inotify` or a similar - mechanism to learn about changed files. - - - During top-down invalidation, the transitive closure of the top-level node is checked and only - those nodes are kept whose transitive closure is clean. This is better if we know that the - current node graph is large, but we only need a small subset of it in the next build: bottom-up - invalidation would invalidate the larger graph of the first build, unlike top-down invalidation, - which just walks the small graph of second build. - -We currently only do bottom-up invalidation. - -To get further incrementality, we use _change pruning_: if a node is invalidated, but upon rebuild, -it is discovered that its new value is the same as its old value, the nodes that were invalidated -due to a change in this node are "resurrected". - -This is useful, for example, if one changes a comment in a C++ file: then the `.o` file generated -from it will be the same, thus, we don't need to call the linker again. - -## Incremental Linking / Compilation - -The main limitation of this model is that the invalidation of a node is an all-or-nothing affair: -when a dependency changes, the dependent node is always rebuilt from scratch, even if a better -algorithm would exist that would mutate the old value of the node based on the changes. A few -examples where this would be useful: - - - Incremental linking - - When a single `.class` file changes in a `.jar`, we could theoretically modify the `.jar` file - instead of building it from scratch again. - -The reason why Bazel currently does not support these things in a principled way (we have some -measure of support for incremental linking, but it's not implemented within Skyframe) is twofold: -we only had limited performance gains and it was hard to guarantee that the result of the mutation -is the same as that of a clean rebuild would be, and Google values builds that are bit-for-bit -repeatable. - -Until now, we could always achieve good enough performance by simply decomposing an expensive build -step and achieving partial re-evaluation that way: it splits all the classes in an app into -multiple groups and does dexing on them separately. This way, if classes in a group do not change, -the dexing does not have to be redone. - -## Restarting SkyFunctions - -Another inefficiency is that, currently, if a `SkyFunction` implementation cannot complete its job -because one of its dependencies is missing, it needs to be completely restarted instead of resuming -where it left off. This is currently not a big problem because we usually learn all the -dependencies after a small amount of work. The only exceptions are package loading and execution of -actions; these are both external processes that are expensive to restart. We allow package loading -to proceed fully, store the loaded package away, record the dependencies in the graph, and on -re-execution of the function return the already loaded package. I.e., we allow the function to keep -state between executions. - -If this turns out to be a significant performance or code health problem, there are alternative ways -to add a more principled mechanism to keep state between executions: - - - Splitting each node into multiple ones so that each smaller node only has to do one round of - dependency discovery (effectively continuation passing); this requires explicit code. - - By reimplementing Skyframe on some sort of lightweight thread infrastructure (e.g. - [Quasar](http://docs.paralleluniverse.co/quasar/)) so that function execution can be suspended - and resumed without a large performance hit and without requiring this to be explicit in the - code. - - By maintaining state for each `SkyFunction` instance between restarts (this is the workaround we - are using for package loading, but is not implemented as a first-class feature of the evaluation - framework). - -## Mapping to Bazel concepts - -This is a rough overview of some of the `SkyFunction` implementations Bazel uses to perform a build: - - - **FileStateValue**. The result of an `lstat()`. For existent files, we also compute additional - information in order to detect changes to the file. This is the lowest level node in the Skyframe - graph and has no dependencies. - - **FileValue**. Used by anything that cares about the actual contents and/or resolved path of a - file. Depends on the corresponding `FileStateValue` and any symlinks that need to be resolved - (e.g. the `FileValue` for `a/b` needs the resolved path of `a` and the resolved path of `a/b`). - The distinction between `FileStateValue` is important because in some cases (for example, - evaluating file system globs (e.g. `srcs=glob(["*/*.java"])`) the contents of the file are not - actually needed. - - **DirectoryListingValue**. Essentially the result of `readdir()`. Depends on the associated - `FileValue` associated with the directory. - - **PackageValue**. Represents the parsed version of a BUILD file. Depends on the `FileValue` of - the associated `BUILD` file, and also transitively on any `DirectoryListingValue` that is used - to resolve the globs in the package (the data structure representing the contents of a `BUILD` - file internally) - - **ConfiguredTargetValue**. Represents a configured target, which is a tuple of the set of actions - generated during the analysis of a target and information provided to configured targets that - depend on this one. Depends on the `PackageValue` the corresponding target is in, the - `ConfiguredTargetValues` of direct dependencies, and a special node representing the build - configuration. - - **ArtifactValue**. Represents a file in the build, be it a source or an output artifacts - (artifacts are almost equivalent to files, and are used to refer to files during the actual - execution of build steps). For source files, it depends on the `FileValue` of the associated - node, for output artifacts, it depends on the `ActionExecutionValue` of whatever action generates - the artifact. - - **ActionExecutionValue**. Represents the execution of an action. Depends on the `ArtifactValues` - of its input files. The action it executes is currently contained within its sky key, which is - contrary to the concept that sky keys should be small. We are working on solving this - discrepancy (note that `ActionExecutionValue` and `ArtifactValue` are unused if we do not run the - execution phase on Skyframe). diff --git a/site/designs/skylark/declared-providers.md b/site/designs/skylark/declared-providers.md deleted file mode 100644 index ef88b5ba3f..0000000000 --- a/site/designs/skylark/declared-providers.md +++ /dev/null @@ -1,288 +0,0 @@ ---- -layout: contribute -title: Declared Providers ---- - -# Design Document: Declared Providers - -**Design documents are not descriptions of the current functionality of Skylark. -Always go to the documentation for current information.** - - -**Status:** Approved - -**Authors:** [Dmitry Lomov](mailto:dslomov@google.com), -[Laurent Le Brun](mailto:laurentlb@google.com) - -**Design document published**: 2016-06-06 - -## Motivation - -Skylark rules use simple Skylark structs as their providers. Skylark providers -are identified as simple names, such as 'java' or 'files'. This approach has the -advantage of simplicity, but as the number and complexity of Skylark rules grow, -we run into engineering scalability problems: - -* Using simple names for providers might lead to name conflicts (when - unrelated rules call their providers the same simple name). -* There is no clear formal way to add documentation for those providers; if - any, the documentation is in prose in rule's doc comment, where it tends to - become obsolete/incomplete; most existing providers have no documentation - explaining their contracts at all. -* It’s hard to know which fields to expect in a provider. -* It’s hard to know which rule can depend on which rule. - -## Goals - -* Solve name-conflict problem for providers -* Allow to specify providers in Skylark rules with the same level of - robustness as other components of the language, such as rules and aspects -* Enable the same or better documentability of Skylark providers as native - providers allow -* Improve providers interoperability with native code. - -## Proposal - -We propose a redesign of how Skylark rules deal with providers to address the -above concerns. The redesign can occur in stages; those stages represent -implementation stages, but allow Skylark users to gradually opt for "more and -more engineering" as their custom rules progress from a small project on the -side to a public release. - -Our proposal is backwards compatible with the existing providers in Bazel and -allows easy, gradual piecemeal replacement of them. - -### Stage 1: Solving the Naming Problem - -Under the new proposal, a minimum implementation of a custom provider looks like -this: - -``` -# rust.bzl - -# Introduces a provider. `rust_provider` is now both a function -# that can be used to construct the provider instance, -# and a "symbol" that can be used to access it. -rust_provider = provider() - -def _impl(ctx): - # `rust_provider` is used as struct-like constructor - # it accepts the same arguments as a standard `struct` function - rust = rust_provider(defines = "-DFOO", ...) - # return value of rule implementation function - # is just a list of providers; their "names" are specified - # by their constructors, see below - return [ctx.provider(files = ...), rust] -rust_library = rule(implementation = _impl, - # Optional declaration; the rule MUST provide all the - # providers in this list - providers = [rust_provider]) -``` - -``` -# Example of how to access the provider - -load(":rust.bzl", "rust_provider") - -def _impl(ctx): - dep = ctx.attr.deps[0] # Target object - # `rust_provider` is used as a key to access a particular - # provider - defines = dep[rust_provider].defines ... -``` - -#### The provider function - -* We introduce two new kinds of Skylark values, a *provider declaration* and a - *provider*. -* Provider declaration (`rust_provider` in the example) is created using the - `provider` function. -* Provider declaration can be used to construct a *provider* (`rust` in the - example). Provider is a struct-like Skylark value, with the only difference - that every provider is associated with its declaration (it is a different - type). Arguments of a provider declaration when used as a function are - exactly the same as that of a built-in `struct` function. -* [Target](http://www.bazel.build/docs/skylark/lib/Target.html) objects become - dictionaries of providers indexed by their declaration. Bracket notation can - be used to retrieve a particular provider. Thus, provider declarations are - [symbol-like](https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Symbol) - values. -* Providers can be private to an extension file; in that case the provider - cannot be accessed outside that file. - -#### Default providers (ctx.provider) - -There is a set of default providers (`files`, `runfiles`, `data_runfiles`, -`executable`, `output_groups`, etc.). We group them in a single provider, -`ctx.provider`: - -``` -defaults = ctx.provider(files = set(), runfiles = ...) -``` - -The current set of APIs on Target objects that access these providers -(`target.files`, `target.output_group("name")` etc.) will continue to work. - -#### Return value - -The implementation function can return either a provider, or a list of -providers. It is an error to return two providers of the same type. - -``` -return [defaults, rust, cc] -return ctx.provider(files = set()) -``` - -#### Declaring providers returned by a rule - -Users need to know which rules provide which providers. This is important for -documentation and for knowing which dependencies are allowed (e.g. we want to -find easily what can go in the deps attribute of cc_library). - -We allow rules to declare the providers they intend to return with a `providers` -argument of a -<code>[rule](http://www.bazel.build/docs/skylark/lib/globals.html#rule)</code> -function. It is an error if the rule implementation function does not return all -the providers listed in `providers`. It may however return additional providers. - -``` -rust_provider = provider() - -rust_library = rule(implementation = _impl, - # Optional declaration; the rule MUST provide all the - # providers in this list - providers = [rust_provider]) -``` - -#### Migration path and support for "legacy" providers - -To support current model of returning providers, where they are identified by a -simple name, we continue to allow providers name in the return struct: - -``` -def _impl(ctx): - ... - return struct( - legacy_provider = struct(...), - files = set(...), - providers = [rust]) -``` - -This also works for “default” providers, such as “files”, “runfiles” etc. -However if one of those legacy names is specified, it is an error to have -ctx.provider instance in the list of `providers`. - -We also allow returning a declared provider both directly and with a simple -name: - -``` -def _impl(ctx): - ... - return struct(rust = rust, providers = [rust]) -``` - -This allows the rules to mix old and new style, and migrate rule definition to a -new style without changing all the uses of that rule. - -Old-style providers with simple names can still be accessed with dot-notation on -Target object, so all of the following is valid. - -Old-style usage: - -* `target.rust` (=> rust) -* `getattr(target, "rust")` (=> rust) -* `hasattr(target, "rust")` (=> True) - -New-style usage: - -* `target[rust_provider]` (=> rust) -* `rust_provider in target` (=> True) -* `target.keys` (=> [rust_provider]) -* `target.values` (=> [rust]) -* `target.items` (=> [(rust_provider, rust)]) - -#### type function - -Type function on providers returns a string `"provider"`. Type function on a -provider instance returns a string `"struct"`. - -### Stage 2: Documentation and Fields - -Provider declarations are a convenient place to add more annotations to -providers. We propose 2 specific things there: - -``` -rust_provider = provider( - doc = "This provider contains Rust information ...", - fields = ["defines", "transitive_deps"] -) -``` - -This specifies documentation for the provider and a list of fields that the -provider can have. - -If `fields` argument is present, extra, undeclared fields are not allowed. - -Both `doc` and `fields` arguments to `provider` function are optional. - -`fields` argument can also be a dictionary (from string to string), in that case -the keys are names of fields, and the values are documentation strings about -individual fields - -``` -rust_provider = provider( - doc = "This provider contains Rust information ...", - fields = { - "defines": "doc for define", - "transitive_deps": "doc for transitive deps, - }) -``` - -### Native Providers - -Providers (as Skylark values) can be also declared natively. A set of -annotations can be developed to facilitate declaring them with little effort. - -As a strawman example: - -``` -/** - * Hypothetical implementation of Skylark provider value (result of - * provider(..) function. - */ -class SkylarkProviderValue extends SkylarkValue { - ... - /** - * Creates a SkylarkProviderValue for a native provider - * `native` must be annotated with @SkylarkProvider annotation. - * Field accessors and constructor function appear magically. - */ - static <T> SkylarkProviderValue forNative(Class<T> native) { ... } -} - -@SkylarkProvider(builder = Builder.class) -// A class with this annotation can be used as provider declaration -class rustProvider implements TransitiveInfoProvider { - @SkylarkProviderField(doc = ...) - // Skylark name is 'defines' - String getDefines() { ... } - - @SkylarkProviderField(doc = ...) - // Skylark name is 'transitive_deps' - NestedSet<Artifact> getTransitiveDeps() { ... } - - @SkylarkProviderField(doc = ...) - // Not allowed, the set of types exposed to Skylark is restricted - DottedVersion getVersion() { ... } - - // Automatically used to provide an implementation for - // construction function. - static class Builder { - // a setter for 'defines' field, based on name. - void setDefines(String defines) { ... } - // a setter for 'transitive_deps' field, based on name. - void setTransitiveDeps(...) {...} - rustProvider build() { ... } - } -} -``` diff --git a/site/designs/skylark/parameterized-aspects.md b/site/designs/skylark/parameterized-aspects.md deleted file mode 100644 index d1c23a1722..0000000000 --- a/site/designs/skylark/parameterized-aspects.md +++ /dev/null @@ -1,307 +0,0 @@ ---- -layout: contribute -title: Parameterized Skylark Aspects ---- - -# Design Document: Parameterized Skylark Aspects - -**Design documents are not descriptions of the current functionality of Skylark. -Always go to the documentation for current information.** - - -**Status:** Approved (Proposal #2), Stage 1 implemented. - -**Author:** [Dmitry Lomov](mailto:dslomov@google.com), -[Lindley French](mailto:lindleyf@google.com) - -**Design document published**: 2016-04-18 - -*Proposal #2 is approved* - -# Motivation - -When rules apply aspects to their dependencies, they often need to -parameterize these aspects with certain values that depend on rule -instances. Typical example: - -* `python_proto_library` rule (just like other `*_proto_library` rules) - need to generate code for different API versions depending on the - attribute py_api_version in the rule instance - - -In general, a different set of parameters for aspects means not only -different actions that aspects generate, but also a different set of -extra dependencies that aspects introduce (for example, depending on -the value of py_api_version, python proto aspect will depend on -different versions of python protobuf runtime library). - -This functionality is already available for native implementations of - aspects. Native aspects can be parameterized with - [AspectParameters](https://github.com/bazelbuild/bazel/blob/72229431c24ad08f0546b03ede9737b633034e30/src/main/java/com/google/devtools/build/lib/packages/AspectParameters.java): (key,value)-dictionaries, where keys and values are simple strings: - -1. AspectParameters are produced by *parameter extractor*: a function - that works on rule instance and produces an aspect parameter dictionary - based on rule instance attribute values - -2. AspectParameters affect both the aspect definitions ([aspect -definition of a particular aspect class depends on AspectParameters](https://github.com/bazelbuild/bazel/blob/f64730fcff20b7d9428e6bd8471ac057ae1bb3b1/src/main/java/com/google/devtools/build/lib/packages/NativeAspectClass.java)) -and aspect implementations (AspectParameters are available to [ConfiguredAspectFactory.create](https://github.com/bazelbuild/bazel/blob/0773430188885e075121ebf720c82bb05a39db21/src/main/java/com/google/devtools/build/lib/analysis/ConfiguredAspectFactory.java#L31)) - -This document describes how to expose this - functionality to Skylark in safe and powerful way. - -# Non-solutions and concerns - -*Too much parameterization is bad*. Consider the following strawman: -why cannot we make the entire originating rule instance available to the -propagating aspect? This is very powerful, but it introduces a -_M\*N work problem_: every rule instance originating an aspect will -generate a completely different aspect! Effectively, every rule -originating an aspect will generate an entirely new graph of transitive -dependencies. - -In the same vein, it is desirable to always limit the parameter space -across which the aspect parameters might vary. -The good design of Skylark aspect parameterization needs to account for that. - -*Using different instances of aspects/rules instead of parameters is -unworkable*. It could be argued that, for example, instead of having -a api_version on python_proto_library, we should have several different -rule classes, py_proto_library_<api version>. This is quite unergonomic. -It is barely bearable for *_proto_library case, and completely -impossible for ndk_cc_library where the potential parameter space is -large (for every parameter combination, a new rule class needs to be -introduced; Skylark macros cannot help here, as Skylark macros cannot -introduce new names into global namespace). - -*Increased potential of action conflict.* As it stands now, aspects -output their artifacts to the output directories of the targets they -apply to. This is fragile as unrelated aspects can generate conflicting -actions, and with introduction of parameters the possibility of that -increases (we now have the possibility of the same aspect with different -parameters being applied to a target; aspect author might forget to -disambiguate carefully, leading to subtle and hard to find bugs). - - -# Solutions - -The primary idea for solving the M*N problem is forcing the aspect -author to limit the parameter space and prohibit its accidental -expansion. Instead of having a direct function RI -> AI (where RI is -a rule instance, AI is an aspect instance), we will have (possibly -indirectly) two functions, RI -> P and P -> AI, where P is a finite set -of possible parameter values defined in advance. - -## Proposal #1 - -We introduce the proposal by example (the discussion is below): - -```python -SUPPORTED_API_VERSIONS = ["1","2","3"] - -def _py_aspect_attrs(api_version): - if api_version = "1": - return { '_protoc' : attr.label(default = "//tools/protoc:v1") } - else if api_version == "2": -…. - -def _py_aspect_impl(target, ctx, params): - if params.api_version == "1": …. -py_proto_aspect = aspect(implementation = _py_aspect_impl, - # params declare all aspect parameters with all possible values - params = { 'api_version' : set(SUPPORTED_API_VERSIONS) }, - attr_aspects = ['deps'], - # rhs of attrs can still be dictionary if no dependencies on params - attrs = _py_aspect_attrs, -) -# Can be omitted, see below. -def _py_proto_library_parameter_extractor(py_api_version, some_other_attr): - return { 'api_version' : str(py_api_version), } -py_proto_library = rule(implementation = _py_proto_library_impl, - attrs = { - 'py_api_version' : attr.int() - 'deps': attr.label_list(aspect = py_proto_aspect, - # Can be omitted: the default extractor - # just strs all rule attributes with the same - # names as aspect parameters. - aspect_parameter_extractor = _py_proto_library_parameter_extractor, - ), - 'some_other_attr' : attr.string(), - } -) -``` - - - -Here are the elements we introduce: - -1. Aspects declare their parameters by means of `params` argument to - `aspect` function. The value of that argument is a dictionary from - parameter name to the set of possible values for that parameter. - We require that the parameter space for an aspect is defined upfront. - We reject any parameter values that are not declared in advance. - In this way we address the M*N work problem: we force the aspect - author to limit the parameter space and prohibit its accidental - expansion. - Note: the better expression for this would have been to require params - to always be of certain enumeration type, but we do not have - enumeration types in Skylark. - -2. We allow aspect attributes (essentially the extra dependencies that - aspects introduce) to depend on aspect parameters. To this end, we - allow functions as values of `attrs` argument for `aspect` function. - If the `attrs` argument is a function it is called with aspect - parameters to obtain the attributes dictionary (the parameters are - guaranteed to be within their specified range i.e. set of values). - If `attrs` argument is a dictionary, it is used as is (compatible - with current behavior). - Note: it is possible to extend `attr_aspects` argument in the same way - as well, if needed. - -3. Parameter dictionary is passed as a third parameter to aspect - implementation function. - -4. When rules specify an aspect to apply to their attribute, they can - optionally specify *a parameter extractor* - a Skylark function that - produces a parameter dictionary based on values of rule attributes. - It is an error when a value of parameter produced by a parameter - extractor is not within its specified range. The default parameter - extractor just stores the values of rule attributes with the same name - as parameters of an aspect in question. - -### Implementation stages for proposal #1 - -*Stage 1.* Make the params available to aspect implementation function. This includes: - -1. Adding `params` argument to `aspect` function. - Declared parameters and their ranges become a part of `SkylarkAspect`. - -2. Adding appropriate parameter extractor (just the default one, - str-ing all the relevant attribute values) and introduce the validation - when creating an aspect in `Attribute.SkylarkRuleAspect` - -3. Passing parameter dictionary to aspect implementation function: - see `SkylarkAspectFactory`. - -*Stage 2.* Parameterize Skylark aspect attributes with aspect -parameters. This involves straightforward changes to `aspect` Skylark -function and to `Attribute.SkylarkRuleAspect`. The only tricky thing -there is handling evaluation exceptions from Skylark. - -*Stage 3.* Implement custom parameter extractors: a straightforward -change to `Attribute.SkylarkRuleAspect` (most of error handling should be -in place by that stage). - -## Proposal #2 (alternative to #1) - -In this proposal, aspect parameters are just aspect’s *explicit* -attributes. We restrict the parameter space by requiring all aspect -explicit attributes to have `values` declaration. - -Here is how the pervious example will look like in this proposal: - -```python -SUPPORTED_API_VERSIONS = ["1","2","3"] - -# For rules, configured default function has access to cfg as well, we -# do not support it in aspects -def _py_aspect_protoc(attr_map): - if attr_map.api_version = "1": - return Label("//tools/protoc:v1") - else if attr_map.api_version "2": - … - -def _py_aspect_impl(target, ctx): - if ctx.attrs.api_version == "1": …. - -py_proto_aspect = aspect(implementation = _py_aspect_impl, - attr_aspects = ['deps'], - attrs = { - # For aspect implicit attributes, we allow computed defaults. - # We still require defaults for all implicit attributes - '_protoc' : attr.label(default = _py_aspect_protoc) - # We allow non-implicit attributes. They MUST declare a range of - # possible values, and they MUST be of a limited set of types - # (initially just strings) - 'api_version' : attr.string(values = SUPPORTED_API_VERSIONS) - } -) - - -# Can be omitted, see below. -def _py_proto_library_parameter_extractor(py_api_version, some_other_attr): - return { 'api_version' : str(py_api_version), } -py_proto_library = rule(implementation = _py_proto_library_impl, - attrs = { - 'py_api_version' : attr.int() - 'deps': attr.label_list(aspect = py_proto_aspect, - # Can be omitted: the default extractor - # just passes all rule attributes with the same - # names as aspect non-implicit attributes - # (aka "parameters"). - aspect_parameter_extractor = _py_proto_library_parameter_extractor, - ), - 'some_other_attr' : attr.string(), - } -) -``` - -Here are the elements we introduce: - -1. We limit the types of explicit aspect attributes to "primitive" values - (strings, ints, booleans). - Note: initially those attributes should just be strings in line with - AspectParameters; if we want more types here, we can extend - AspectParameters to support more types. - -2. To facilitate parameterizing aspect dependencies, we allow *implicit* - aspect attributes to have computed defaults, exposed in the same way - computed defaults are exposed to Skylark rules: "default value" of - an attribute can be a function that computes the value given - an attribute map. - Note: computed default functions for Skylark rules have access to - configuration information as well. We cannot support this for aspects - at the moment; we need to clarify the relationship between aspects and - configurations, so this is TBD. - -3. When rules specify an aspect to apply to their attribute, they can - optionally specify *a parameter extractor* - a Skylark function that - produces a parameter dictionary based on values of rule attributes. - The keys of the computed dictionary must match the names of all - non-explicit attributes on the aspect. It is an error when a value of - parameter produced by a parameter extractor is not within its specified - range. The default parameter extractor just passes values of rule - attributes with the same name as explicit attributes of an aspect - in question. - -### Implementation stages for proposal #2 - -(Those stages correspond to implementation stages for proposal #1: at their completion, the same functionality becomes available) - -*Stage 1.* Allow explicit attributes with values restriction on aspects: - -1. Modify `aspect` value. - -2. Add appropriate parameter extractor (just the default one, - passing through all the relevant attribute values) and introduce the - validation when creating an aspect in `Attribute.SkylarkRuleAspect`. - -3. Ensure that explicit attribute values are passed through to aspect - implementation function: see `SkylarkAspectFactory` - -Stage 1 is [impemented](https://github.com/bazelbuild/bazel/commit/74558fcc8953dec64c2ba5920c8f7a7e3ada36ab). - -*Stage 2.* Allow computed defaults for aspect’s implicit attributes. -This involves changes to `aspect` Skylark function and to -`Attribute.SkylarkRuleAspect`. There are two non-obvious parts: - -1. we should not allow computed defaults to be default values of - attributes after AspectDefintion is computed - (i.e. `SkylarkAspect.getDefinition`) - -2. proper error handling is needed here. - -*Stage 3.* Implement custom parameter extractors: a straightforward -change to Attribute.SkylarkRuleAspect (most of error handling should -be in place by that stage). - diff --git a/site/designs/skylark/saner-skylark-sets.md b/site/designs/skylark/saner-skylark-sets.md deleted file mode 100644 index 752f8d6a66..0000000000 --- a/site/designs/skylark/saner-skylark-sets.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -layout: contribute -title: Saner Skylark Sets ---- - -# Design Document: Saner Skylark Sets - -### (Sacrificing Superfluous Safety) - -**Design documents are not descriptions of the current functionality of Skylark. -Always go to the documentation for current information.** - - -**Status:** Draft - -**Author:** [Dmitry Lomov](mailto:dslomov@google.com) - -**Design document published**: 2016-07-25 - -## Motivation - -NestedSets (an implementation used in Skylark `set` data type) is an essential -data structure for passing transitive cumulative data during Bazel's analysis. -The reason for that is its memory efficiency: a union of two sets carries a -constant memory overhead (compare that to lists, for example, where a union of -two lists has an O(length of original lists) memory overhead). - -When Bazel builds a dependency graph, all data returned by providers coming from -rule implementation stays in memory roughly throughout the lifetime of a Bazel -server. If providers use lists for their cumulative data, the total amount of -memory consumed by them will grow as O(N^2) where N is the number of nodes in -the build graph. Sets are the only data structure available in Skylark that can -reduce that amount to O(N). - -However, sets in Skylark suffer from several deficiencies, which preclude their -usage in providers, which leads in turn to performance issues. - -* Skylark sets cannot contain structs (although they can contain tuples, which - is used sometimes to overcome that limitation: - [example](https://github.com/bazelbuild/bazel/blob/a48e8e3db5a149777c2887fc7fc572837dd0ac1e/src/test/java/com/google/devtools/build/lib/ideinfo/intellij_info.bzl#L84)) -* Only sets of primitive types (not even tuples) are allowed in rule providers - -This document discusses the reasons for these deficiencies and suggests several -ways to resolve them. - -## Understanding Current Behavior - -### Typing Skylark sets - -Skylark sets are, in a certain sense, typed. Heterogeneous Skylark sets (e.g. -ints and strings in the same set) are not allowed. Skylark set carries its -contentType (the type of its elements) with it. - -First of all, all elements of Skylark set must be immutable. Sets of sets are -not allowed either. - -Second, when a union of skylark sets is computed, their contentTypes are -*intersected* to find a type that contains elements of both sets. Union -operation fails if that intersection is empty. - -Type intersection disallows heterogeneous sets since intersection of two -primitive types is empty unless it is one and the same type. However, all tuples -in Skylark have the same type, TUPLE, therefore sets of heterogeneous tuples, -and even tuples of different length, are allowed. Also, since all tuples share -the same type, Skylark set has no information about components of its -constituent tuples. - -### Providers and safe values - -All values returned as providers from rule and aspect implementations are -required to be *safe*. Safe values are defined as follows (note that the notion -is defined on values, not on types): - -* Primitive safe values are ints, strings, booleans, Files (Artifacts), Labels - and native providers (TransitiveInfoCollections) -* Lists of safe values are safe -* Tuples of safe values are safe -* Dictionaries are safe if their keys and values are safe -* Sets are safe if their contentType is that of primitive safe value. - -Note that in this entire definition, only the definition of safe sets involves -any types. For all other composite values, their constituents are examined -directly, but in case of sets, this is too expensive (because sets supposedly -hold transitive cumulative information), so their content type is examined -instead. - -This poses a problem if we want to allow safe sets of tuples, as set of tuples -forget what are the types of constituents of those tuples. - -## What needs to change - -As discussed in the "Motivation" section, we need to make sure that Skylark -rules and aspects can use sets to pass information during the build analysis; -that is we need to allow two things: - -* sets should be allowed to contain structs -* sets of structs should be allowed in rule providers - -### Equality and mutability of structs - -Allowing sets to contain structs will require equality semantics on structs to -change (from reference to structural equality). We deem the risk of this change -very low; even if structs are compared for equality now in user code, that is -likely a bug, and since structs are immutable, the observed behavior should not -change. - -Structs, just like tuples, can be (deeply) mutable and immutable: - -``` -l = [27, 42, 30] # This list is mutable (inside a function definition) -t = (l, 42) # This tuple is mutable, since l can be modified -s = struct(field = l) # This struct is mutable, since l can be modified -``` - -Only immutable structs will be allowed as elements of sets, similarly to how -tuples are handled today. - -### Provider value safety - -To allow sets of structs inside providers, we need to reconcile value safety -check with set typing (Recall that sets forget what constituents their elements -have, so fast safety check is impossible). - -#### Alternative #1: Higher-fidelity typing for sets - -We can record more precise element types in sets (not just STRUCT, but the -entire list of fields of those structs). We will need to define what does struct -intersection mean (for example, do we allow sets of structs with disjoint -collections of fields?). If we want to record types for sets of tuples, we will -need to have a very permissive type system to preserve current behavior. For any -restriction we introduce, we need a careful rationale. - -#### Alternative #2: Only allow sets of declared providers with type information - -If we implement "[Declared providers](/designs/skylark/declared-providers.html)" -proposal and provide more extesive type information on top of that, sets can get -typing information from there. - -#### Alternative #3: Abandon safe value check - -Looking at all the issues in this document, it appears that safe value check -causes more trouble than the value it brings. The only substantive requirement -for provider values is that they are immutable. Another motivation for a safe -value check was serialization, but if we are serious about serializing -configured targets graph, we will need to learn to serialize arbitrary Skylark -values anyway. diff --git a/site/designs/skylark/skylark-design-process.md b/site/designs/skylark/skylark-design-process.md deleted file mode 100644 index 52d3773ba3..0000000000 --- a/site/designs/skylark/skylark-design-process.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -layout: contribute -title: Skylark Design Review Process ---- - -# Skylark Design Review Process - -Authors: [Dmitry Lomov](mailto:dslomov@google.com), -[Laurent Le Brun](mailto:laurentlb@google.com), [Florian Weikert](mailto:fwe@google.com) - - -## Motivation - -As Bazel adoption grows, we are seeing both more requests for adding -features to Skylark as a language, and for exposing more and more -functionality available in Bazel internally to Skylark extensions. -These requests originate both within and outside the Bazel team. As -their number grows (and we expect them to grow more and more over time), -addressing those requests meets several challenges: - -* We need to keep Skylark as a language (and a set of -associated APIs/libraries) concise and consistent, easy to learn -and use, and documented. - -* Any APIs or solutions we adopt will be painful to change -down the road, so the more consistent, orthogonal, and open -to future extensions they are, the less pain we and our users -will encounter. - -* It is difficult for engineers - both within the team and outside - -to approach making changes to Skylark. As people attempt to do so, -they experience a lot of friction: patches get written and the discussion -starts where reviewers and the author attempt to solve Skylark -API issues, code design issues, implementation issues, compatibility -issues and so on and so forth all in one code review thread. -The result is friction and frustration, and the quality of the end result -is not guaranteed. - -This document proposes an informal but orderly and well-defined process -for Skylark language changes to address the above challenges. - -## Goals - -* Facilitate healthy growth of Skylark as a language - -* Ensure that Skylark the language has a clear set of experts caring for its development - -* Reduce friction for engineers exposing APIs to Skylark or proposing new Skylark features - -## Non-goals - -* Replace general design review process in Bazel team. -The process described here is strictly about the Skylark language and APIs. -In particular, changes to native rules that do not affect Skylark as a -language much, such as adding/removing an attribute, or adding a new -native rule, are not required to go through this process -(whereas exposing a new provider to Skylark from a native rule is). - -## The Process - -Changes to Skylark, both the language and the exposed APIs, -are evaluated by the Skylark Reviewers Panel. - - -1. The **author** of the proposed change sends the design document to the -[bazel-dev@googlegroups.com] mailing list with a subject containing -"SKYLARK DESIGN REVIEW" - -1. Design doc can be either of: - 1. A universally-commentable Google Doc - 1. A Gerrit code review request with a design doc in - [Markdown](https://guides.github.com/features/mastering-markdown/) - format. - -1. The design doc should include: - 1. Motivation for the change (a GitHub issue link is acceptable) - 2. An example of the usage of the proposed API/language feature - 3. Description of the proposed change - - [A model example design doc](/designs/skylark/parameterized-aspects.html) - (although that is probably an overkill). - -1. **Skylark Reviewers** respond to a document within *2 business days* - -1. **Skylark Reviewers** are responsible for bringing in - **subject-matter experts** as needed (for example, a change involving - exposing a certain provider from cc_library rule should be reviewed by - C++ rules expert as well) - -1. **Skylark Reviewers** facilitate the discussion and aim to reach -a "looks good/does not look good" decision within *10 business days*. - -1. **Skylark Reviewers** operate by consensus. diff --git a/site/docs/README.md b/site/docs/README.md deleted file mode 100644 index a13f46bf46..0000000000 --- a/site/docs/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Our documentation is in another directory. - -Bazel's documentation is now versioned. This directory contains redirects to -the latest version of each page. - -To make changes to Bazel documentation at HEAD, please edit the files under -`versions/master`. Once a release is cut, the docs in the `master` directory -will be snapshotted and copied into a versioned directory for the release. - -Happy documenting! diff --git a/site/versions/master/docs/bazel-overview.md b/site/docs/bazel-overview.md index 76c07c965d..76c07c965d 100644 --- a/site/versions/master/docs/bazel-overview.md +++ b/site/docs/bazel-overview.md diff --git a/site/docs/bazel-user-manual.md b/site/docs/bazel-user-manual.md deleted file mode 100644 index f9fea184a6..0000000000 --- a/site/docs/bazel-user-manual.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/bazel-user-manual.html ---- diff --git a/site/docs/be/docker.md b/site/docs/be/docker.md deleted file mode 100644 index 9279688d74..0000000000 --- a/site/docs/be/docker.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/be/docker.html ----
\ No newline at end of file diff --git a/site/docs/be/pkg.md b/site/docs/be/pkg.md deleted file mode 100644 index b5cdea0d76..0000000000 --- a/site/docs/be/pkg.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/be/pkg.html ----
\ No newline at end of file diff --git a/site/docs/best-practices.md b/site/docs/best-practices.md index e936d3112e..ca360954b9 100644 --- a/site/docs/best-practices.md +++ b/site/docs/best-practices.md @@ -1,4 +1,321 @@ --- -layout: redirect -redirect: docs/best-practices.html +layout: documentation +title: Best practices --- + +# Best practices for Bazel + +This document assumes that you are familiar with Bazel and provides advice on structuring your +projects to take full advantage of Bazel's features. + +The overall goals are: + +- To use fine-grained dependencies to allow parallelism and incrementality. +- To keep dependencies well-encapsulated. +- To make code well-structured and testable. +- To create a build configuration that is easy to understand and maintain. + +These guidelines are not requirements: few projects will be able to adhere to all of them. As the +man page for lint says, "A special reward will be presented to the first person to produce a real +program that produces no errors with strict checking." However, incorporating as many of these +principles as possible should make a project more readable, less error-prone, and faster to build. + +This document uses the requirement levels described in +[this RFC](https://www.ietf.org/rfc/rfc2119.txt). + +## Contents + +- [General structure](#general-structure) + - [Running builds and tests](#running-builds-and-tests) + - [Third party dependencies](#third-party-dependencies) + - [Depending on binaries](#depending-on-binaries) + - [Versioning](#versioning) + - [.bazelrc](#bazelrc) + - [Packages](#packages) +- [BUILD files](#build-files) + - [BUILD file style guide](#build-file-style-guide) + - [Formatting](#formatting) + - [References to targets in the current package](#references-to-targets-in-the-current-package) + - [Target naming](#target-naming) + - [Visibility](#visibility) + - [Dependencies](#dependencies) + - [Globs](#globs) +- [Skylark](#skylark) + - [Skylark style guide](#skylark-style-guide) + - [Packaging rules](#packaging-rules) + - [Rule choice](#rule-choice) +- [WORKSPACE files](#workspace-files) + - [Repository rules](#repository-rules) + - [Custom BUILD files](#custom-build-files) + - [Skylark repository rules](#skylark-repository-rules) +- [Java](#java) + - [Directory structure](#directory-structure) + - [BUILD files](#build-files) +- [C++](#c) + - [BUILD files](#build-files) + - [Include paths](#include-paths) +- [Protos](#protos) + - [Recommended Code Organization](#recommended-code-organization) + +# General structure + +## Running builds and tests + +A project should always be able to run `bazel build //...` and `bazel test //...` successfully on +its stable branch. Targets that are necessary but do not build under certain circumstances (e.g., +require specific build flags, do not build on a certain platform, require license agreements) +should be tagged as specifically as possible (e.g., "`requires-osx`"). This tagging allows +targets to be filtered at a more fine-grained level than the "manual" tag and allows someone +inspecting the BUILD file to understand what a target's restrictions are. + +## Third party dependencies + +Prefer declaring third party dependencies as remote repositories in the WORKSPACE file. If it's +necessary to check third party dependencies into your repository, put them in a directory called +`third_party/` under your workspace directory. Note that all BUILD files in `third_party/` must +include [license](https://bazel.build/versions/master/docs/be/functions.html#licenses) +declarations. + +## Depending on binaries + +Everything should be built from source whenever possible. Generally this means that, instead of +depending on a library `some-library.so`, you'd create a BUILD file and build `some-library.so` +from its sources, then depend on that target. + +Building from source prevents a build from using a library that was built with incompatible flags +or a different architecture. There are also some features like coverage, static analysis, or +dynamic analysis that will only work on the source. + +## Versioning + +Prefer building all code from head whenever possible. When versions must be used, avoid including +the version in the target name (e.g., `//guava`, not `//guava-20.0`). This naming makes the library +easier to update (only one target needs to be updated). It is also more resilient to diamond +dependency issues: if one library depends on `guava-19.0` and one depends on `guava-20.0`, you +could end up with a library that tries to depend on two different versions. If you created a +misleading alias to point both targets to one guava library, then the BUILD files are misleading. + +## `.bazelrc` + +For project-specific options, use the configuration file `_your-workspace_/tools/bazel.rc`. + +For options that you **do not** want to check into source control, create the configuration file +`_your-workspace_/.bazelrc` and add `.bazelrc` to your `.gitignore`. Note that this file has a +different name than the file above (`bazel.rc` vs `.bazelrc`). + +## Packages + +Every directory that contains buildable files should be a package. If a BUILD file refers to files +in subdirectories (e.g., `srcs = ["a/b/C.java"]`) it is a sign that a BUILD file should be added to +that subdirectory. The longer this structure exists, the more likely circular dependencies will be +inadvertently created, a target's scope will creep, and an increasing number of reverse +dependencies will have to be updated. + +# BUILD files + +## BUILD file style guide + +See the [BUILD file style +guide](https://bazel.build/versions/master/docs/skylark/build-style.html). + +## Formatting + +[Buildifier](https://github.com/bazelbuild/buildifier) should be used to achieve the correct +formatting for BUILD files. Editors should be configured to automatically format BUILD files on +save. Humans should not try to format BUILD files themselves. + +If there is a question as to what the correct formatting is, the answer is "how buildifier formats +it." + +## References to targets in the current package + +Files should be referred to by their paths relative to the package directory (without ever using +up-references, such as `..`). Generated files should be prefixed with "`:`" to indicate that they +are not sources. Source files should not be prefixed with `:`. Rules should be prefixed with `:`. +For example, assuming `x.cc` is a source file: + +```python +cc_library( + name = "lib", + srcs = ["x.cc"], + hdrs = [":gen-header"], +) + +genrule( + name = "gen-header", + srcs = [], + outs = ["x.h"], + cmd = "echo 'int x();' > $@", +) +``` + +## Target naming + +Target names should be descriptive. If a target contains one source file, the target should +generally be named after that source (e.g., a `cc_library` for `chat.cc` should be named "`chat`"). + +The eponymous target for a package (the target with the same name as the containing directory) +should provide the functionality described by the directory name. If there is no such target, do +not create an eponymous target. + +Prefer using the short name when referring to an eponymous target (`//x` instead of `//x:x`). If +you are in the same package, prefer the local reference (`:x` instead of `//x`). + +## Visibility + +Do not set the default visibility of a package to `//visibility:public`. `//visibility:public` +should be individually set for targets in the project's public API. These could be libraries which +are designed to be depended on by external projects or binaries that could be used by an external +project's build process. + +Otherwise, visibility should be scoped as tightly as possible, while still allowing access by tests +and reverse dependencies. Prefer using `__pkg__` to `__subpackages__`. + +## Dependencies + +Dependencies should be restricted to direct dependencies (dependencies needed by the sources listed +in the rule). Do not list transitive dependencies. + +Package-local dependencies should be listed first and referred to in a way compatible with the +[References to targets in the current package](#references-to-targets-in-the-current-package) +section above (not by their absolute package name). + +## Globs + +Do not use recursive globs (e.g., `glob(["**/*.java"])`). Recursive globs make BUILD files +difficult to read, as they skip subdirectories containing BUILD files. Non-recursive globs are +generally acceptable, see language-specific advice below for details. + +Indicate "no targets" with `[]`. Do not use a glob that matches nothing: it is more error-prone and +less obvious than an empty list. + +# Skylark + +## Skylark style guide + +See the [Style guide for .bzl +files](https://bazel.build/versions/master/docs/skylark/bzl-style.html) for Skylark rule guidelines. + +## Packaging rules + +See [Packaging rules](https://bazel.build/versions/master/docs/skylark/deploying.html) for advice +on how to structure and where to put new Skylark rules. + +## Rule choice + +When using a language for which Bazel has built-in rules (e.g., C++), prefer using these rules to +writing your own in Skylark. These rules are documented in the [build +encyclopedia](https://bazel.build/versions/master/docs/be/overview.html). + +# WORKSPACE files + +## Repository rules + +Prefer `http_archive` and `new_http_archive` to `git_repository`, `new_git_repository`, and +`maven_jar`. + +`git_repository` depends on jGit, which has several unpleasant bugs, and `maven_jar` uses Maven's +internal API, which generally works but is less optimized for Bazel than `http_archive`'s +downloader logic. Track the following issues filed to remediate these problems: + +- [Use `http_archive` as `git_repository`'s + backend.](https://github.com/bazelbuild/bazel/issues/2147) +- [Improve `maven_jar`'s backend.](https://github.com/bazelbuild/bazel/issues/1752) + +Do not use `bind()`. See "[Consider removing +bind](https://github.com/bazelbuild/bazel/issues/1952)" for a long discussion of its issues and +alternatives. + +## Custom BUILD files + +When using a `new_` repository rule, prefer to specify `build_file_content`, not `build_file`. + +## Skylark repository rules + +A Skylark repository rule should generally be responsible for: + +- Detecting system settings and writing them to files. +- Finding resources elsewhere on the system. +- Downloading resources from URLs. +- Generating or symlinking BUILD files into the external repository directory. + +Avoid using `repository_ctx.execute` when possible. For example, when using a non-Bazel C++ +library that has a build using Make, it is preferable to use `respository_ctx.download()` and then +write a BUILD file that builds it, instead of running `ctx.execute(["make"])`. + +# Java + +## Directory structure + +Prefer Maven's standard directory layout (sources under `src/main/java`, tests under +`src/test/java`). + +## BUILD files + +Use one BUILD file per package containing Java sources. Every BUILD file should contain one +`java_library` rule that looks like this: + +```python +java_library( + name = "directory-name", + srcs = glob(["*.java"]), + deps = [...], +) +``` + +The name of the library should be the name of the directory containing the BUILD file. The sources +should be a non-recursive glob of all Java files in the directory. + +Tests should be in a matching directory under `src/test` and depend on this library. + +# C++ + +## BUILD files + +Each BUILD file should contain one `cc_library` rule target per compilation unit in the directory. +C++ libraries should be as fine-grained as possible to provide as much incrementality as possible. + +If there is a single source file in `srcs`, the library should be named based on that C++ file's +name. This library should contain a C++ file(s), any matching header file(s), and the library's +direct dependencies. For example, + +```python +cc_library( + name = "mylib", + srcs = ["mylib.cc"], + hdrs = ["mylib.h"], + deps = [":lower-level-lib"] +) +``` + +There should be one `cc_test` rule target per `cc_library` target in the file. The `cc_test`'s +source should be a file named `[libname]_test.cc`. For example, a test for the target above might +look like: + +``` +cc_test( + name = "mylib_test", + srcs = ["mylib_test.cc"], + deps = [":mylib"] +) +``` + +## Include paths + +All include paths should be relative to the workspace directory. Use `includes` only if a public +header needs to be widely used at a non-workspace-relative path (for legacy or `third_party` code). +Otherwise, prefer to use the `copts` attribute, not the `includes` attribute. + +Using `cc_inc_library` is discouraged, prefer `copts` or `includes`. +See [the design document](https://docs.google.com/document/d/18qUWh0uUiJBv6ZOySvp6DEV0NjVnBoEy-r-ZHa9cmhU/edit#heading=h.kmep1cl5ym9k) +on C++ include directories for reasoning. + +# Protos + +## Recommended Code Organization + +- One `proto_library` rule per `.proto` file. +- A file named `foo.proto` will be in a rule named `foo_proto`, which is located in the same + package. +- A `[language]_proto_library` that wraps a `proto_library` named `foo_proto` should be called + `foo_[language]_proto`, and be located in the same package. diff --git a/site/versions/master/docs/blaze-query-v2.html b/site/docs/blaze-query-v2.html index dbe80e4846..dbe80e4846 100644 --- a/site/versions/master/docs/blaze-query-v2.html +++ b/site/docs/blaze-query-v2.html diff --git a/site/versions/master/docs/blaze-user-manual.html b/site/docs/blaze-user-manual.html index 782f5ef7b9..782f5ef7b9 100644 --- a/site/versions/master/docs/blaze-user-manual.html +++ b/site/docs/blaze-user-manual.html diff --git a/site/versions/master/docs/build-ref.html b/site/docs/build-ref.html index 96521c09e5..96521c09e5 100644 --- a/site/versions/master/docs/build-ref.html +++ b/site/docs/build-ref.html diff --git a/site/docs/build-ref.md b/site/docs/build-ref.md deleted file mode 100644 index e08473d1f5..0000000000 --- a/site/docs/build-ref.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/build-ref.html ---- diff --git a/site/docs/cpp.md b/site/docs/cpp.md deleted file mode 100644 index 10cf714df9..0000000000 --- a/site/docs/cpp.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/tutorial/cpp.html ----
\ No newline at end of file diff --git a/site/versions/master/docs/docs_style.css b/site/docs/docs_style.css index bb225e1fd9..bb225e1fd9 100644 --- a/site/versions/master/docs/docs_style.css +++ b/site/docs/docs_style.css diff --git a/site/docs/external.md b/site/docs/external.md index ec45348162..dae89f88bc 100644 --- a/site/docs/external.md +++ b/site/docs/external.md @@ -1,4 +1,203 @@ --- -layout: redirect -redirect: docs/external.html +layout: documentation +title: External Dependencies --- + +# Working with external dependencies + +External dependencies can be specified in the `WORKSPACE` file of the +[workspace directory](/docs/build-ref.html#workspace). This `WORKSPACE` file +uses the same syntax as BUILD files, but allows a different set of +rules. The full list of rules are in the Build Encyclopedia's +[Workspace Rules](/docs/be/workspace.html). + +External dependencies are all downloaded and symlinked under a directory named +`external`. You can see this directory by running: + +``` +ls $(bazel info output_base)/external +``` + +Note that running `bazel clean` will not actually delete the external +directory. To remove all external artifacts, use `bazel clean --expunge`. + +## Supported types of external dependencies + +A few basic types of external dependencies can be used: + +- [Dependencies on other Bazel projects](#bazel-projects) +- [Dependencies on non-Bazel projects](#non-bazel-projects) +- [Dependencies on external packages](#external-packages) + +<a name="bazel-projects"></a> +### Depending on other Bazel projects + +If you want to use targets from a second Bazel project, you can +use +[`local_repository`](http://bazel.build/docs/be/workspace.html#local_repository), +[`git_repository`](https://bazel.build/docs/be/workspace.html#git_repository) +or [`http_archive`](http://bazel.build/docs/be/workspace.html#http_archive) +to symlink it from the local filesystem, reference a git repository or download +it (respectively). + +For example, suppose you are working on a project, `my-project/`, and you want +to depend on targets from your coworker's project, `coworkers-project/`. Both +projects use Bazel, so you can add your coworker's project as an external +dependency and then use any targets your coworker has defined from your own +BUILD files. You would add the following to `my_project/WORKSPACE`: + +```python +local_repository( + name = "coworkers-project", + path = "/path/to/coworkers-project", +) +``` + +If your coworker has a target `//foo:bar`, your project can refer to it as +`@coworkers-project//foo:bar`. + +<a name="non-bazel-projects"></a> +### Depending on non-Bazel projects + +Rules prefixed with `new_` (e.g., +[`new_local_repository`](http://bazel.build/docs/be/workspace.html#new_local_repository), +[`new_git_repository`](https://bazel.build/docs/be/workspace.html#new_git_repository) +and [`new_http_archive`](http://bazel.build/docs/be/workspace.html#new_http_archive) +) allow you to create targets from projects that do not use Bazel. + +For example, suppose you are working on a project, `my-project/`, and you want +to depend on your coworker's project, `coworkers-project/`. Your coworker's +project uses `make` to build, but you'd like to depend on one of the .so files +it generates. To do so, add the following to `my_project/WORKSPACE`: + +```python +new_local_repository( + name = "coworkers-project", + path = "/path/to/coworkers-project", + build_file = "coworker.BUILD", +) +``` + +`build_file` specifies a BUILD file to overlay on the existing project, for +example: + +```python +java_library( + name = "some-lib", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) +``` + +You can then depend on `@coworkers-project//:some-lib` from your project's BUILD +files. + +<a name="external-packages"></a> +### Depending on external packages + +#### Maven repositories + +Use the rule [`maven_jar`](https://bazel.build/versions/master/docs/be/workspace.html#maven_jar) +(and optionally the rule [`maven_server`](https://bazel.build/versions/master/docs/be/workspace.html#maven_server)) +to download a jar from a Maven repository and make it available as a Java +dependency. + +## Fetching dependencies + +By default, external dependencies are fetched as needed during `bazel build`. If +you would like to disable this behavior or prefetch dependencies, use +[`bazel fetch`](http://bazel.build/docs/bazel-user-manual.html#fetch). + +## Using Proxies + +Bazel will pick up proxy addresses from the `HTTPS_PROXY` and `HTTP_PROXY` +environment variables and use these to download HTTP/HTTPS files (if specified). + +<a name="transitive-dependencies"></a> +## Transitive dependencies + +Bazel only reads dependencies listed in your `WORKSPACE` file. If your project +(`A`) depends on another project (`B`) which list a dependency on a third +project (`C`) in its `WORKSPACE` file, you'll have to add both `B` +and `C` to your project's `WORKSPACE` file. This requirement can balloon the +`WORKSPACE` file size, but hopefully limits the chances of having one library +include `C` at version 1.0 and another include `C` at 2.0. + +## Generate a `WORKSPACE` file + +Bazel provides a tool to help generate these expansive `WORKSPACE` files, called +`generate_workspace`. This tool is not included with the binary installer, so +you'll need to clone the [GitHub repo](https://github.com/bazelbuild/bazel) to +use it. We recommend using the tag corresponding to your current version of +bazel, which you can check by running `bazel version`. + +`cd` to the GitHub clone, `git checkout` the appropriate tag, and run the +following to build the tool and see usage: + +``` +bazel run //src/tools/generate_workspace +``` + +Note that you need run this command from your Bazel source folder even if you +build your binary from source. + +You can specify directories containing Bazel projects (i.e., directories +containing a `WORKSPACE` file), Maven projects (i.e., directories containing a +`pom.xml` file), or Maven artifact coordinates directly. For example: + +```bash +$ bazel run //src/tools/generate_workspace -- \ +> --maven_project=/path/to/my/project \ +> --bazel_project=/path/to/skunkworks \ +> --bazel_project=/path/to/teleporter/project \ +> --artifact=groupId:artifactId:version \ +> --artifact=groupId:artifactId:version +Wrote: +/tmp/1437415510621-0/2015-07-20-14-05-10.WORKSPACE +/tmp/1437415510621-0/2015-07-20-14-05-10.BUILD +``` + +The `WORKSPACE` file will contain the transitive dependencies of the given +projects and artifacts. The `BUILD` file will contain a single target, +`transitive-deps`, that contains all of the dependencies. You can copy these +files to your project and add `transitive-deps` as a dependency of your `java_` +targets in `BUILD` files. + +If you specify multiple Bazel projects, Maven projects, or artifacts, they will +all be combined into one `WORKSPACE` file (e.g., if the Bazel project depends on +junit and the Maven project also depends on junit, junit will only appear once +as a dependency in the output). + +You may wish to curate the generated `WORKSPACE` file to ensure it is using the +correct version of each dependency. If several different versions of an artifact +are requested (by different libraries that depend on it), then +`generate_workspace` chooses a version and annotates the `maven_jar` with the +other versions requested, for example: + +```python +# org.springframework:spring:2.5.6 +# javax.mail:mail:1.4 +# httpunit:httpunit:1.6 wanted version 1.0.2 +# org.springframework:spring-support:2.0.2 wanted version 1.0.2 +# org.slf4j:nlog4j:1.2.24 wanted version 1.0.2 +maven_jar( + name = "javax_activation_activation", + artifact = "javax.activation:activation:1.1", +) +``` + +The example above indicates that `org.springframework:spring:2.5.6`, +`javax.mail:mail:1.4`, `httpunit:httpunit:1.6`, +`org.springframework:spring-support:2.0.2`, and `org.slf4j:nlog4j:1.2.24` +all depend on javax.activation. However, two of these libraries wanted +version 1.1 and three of them wanted 1.0.2. The `WORKSPACE` file is using +version 1.1, but that might not be the right version to use. + +You may also want to break `transitive-deps` into smaller targets, as it is +unlikely that all of your targets depend on the transitive closure of your +maven jars. + +## Caching of external dependencies + +Bazel caches external dependencies and re-downloads or updates them when +the `WORKSPACE` file changes. diff --git a/site/docs/getting-started.md b/site/docs/getting-started.md index 1e69dce256..66a87e40c1 100644 --- a/site/docs/getting-started.md +++ b/site/docs/getting-started.md @@ -1,4 +1,97 @@ --- -layout: redirect -redirect: docs/getting-started.html +layout: documentation +title: Getting Started --- + +# Getting Started with Bazel + +## Setup + +Use the [installation instructions](/docs/install.html) to install a copy of +Bazel on your machine. + +## Using a Workspace + +All Bazel builds take place in a [_workspace_](/docs/build-ref.html#workspaces), +a directory on your filesystem that contains source code for the software you +want to build, as well symbolic links to directories that contain the build +outputs (for example, `bazel-bin` and `bazel-out`). The location of the +workspace directory is not significant, but it must contain a file called +`WORKSPACE` in the top-level directory; an empty file is a valid workspace. +The `WORKSPACE` file can be used to reference +[external dependencies](/docs/external.html) required to build the outputs. +One workspace can be shared among multiple projects if desired. + +```bash +touch WORKSPACE +``` + +## Creating a Build File + +To know which targets can be built in your project, Bazel inspects `BUILD` +files. They are written in Bazel's build language which is syntactically +similar to Python. Usually they are just a sequence of declarations of rules. +Each rule specifies its inputs, outputs, and a way to compute the outputs from +the inputs. + +The rule probably most familiar to people who have used `Makefile`s before (as +it is the only rule available there) is the +[genrule](/docs/be/general.html#genrule), which specifies how the output can +be generated by invoking a shell command. + +``` +genrule( + name = "hello", + outs = ["hello_world.txt"], + cmd = "echo Hello World > $@", +) +``` + +The shell command may contain [Make variables](/docs/be/make-variables.html). + +Using the above `BUILD` file, you can ask Bazel to generate the target. + +``` +$ bazel build :hello +. +INFO: Found 1 target... +Target //:hello up-to-date: + bazel-genfiles/hello_world.txt +INFO: Elapsed time: 2.255s, Critical Path: 0.07s +``` + +We note two things. First, targets are normally referred to by their +[label](/docs/build-ref.html#labels), which is specified by the +[name](/docs/be/general.html#genrule.name) attribute of the rule. (Referencing +them by the output file name is also possible, but this is not the preferred +way.) Second, Bazel puts the generated files into a separate directory (the +`bazel-genfiles` directory is actually a symbolic link) so as not to pollute +your source tree. + +Rules may use the output of other rules as input, as in the following +example. Again, the generated sources are referred to by their label. + +``` +genrule( + name = "hello", + outs = ["hello_world.txt"], + cmd = "echo Hello World > $@", +) + +genrule( + name = "double", + srcs = [":hello"], + outs = ["double_hello.txt"], + cmd = "cat $< $< > $@", +) +``` + +Finally, note that, while the [genrule](/docs/be/general.html#genrule) might +seem familiar, it usually is _not_ the best rule to use. It is preferrable to +use one of the specialized [rules](/docs/be/overview.html#rules) for various +languages. + +# Next Steps + +Next, check out the tutorial on building [Java](/docs/tutorial/java.html) +or [C++](/docs/tutorial/cpp.html) programs. diff --git a/site/versions/master/docs/install-compile-source.md b/site/docs/install-compile-source.md index 453d5658df..453d5658df 100644 --- a/site/versions/master/docs/install-compile-source.md +++ b/site/docs/install-compile-source.md diff --git a/site/versions/master/docs/install-os-x.md b/site/docs/install-os-x.md index 249524b91e..249524b91e 100644 --- a/site/versions/master/docs/install-os-x.md +++ b/site/docs/install-os-x.md diff --git a/site/versions/master/docs/install-ubuntu.md b/site/docs/install-ubuntu.md index e2e5d5aa97..e2e5d5aa97 100644 --- a/site/versions/master/docs/install-ubuntu.md +++ b/site/docs/install-ubuntu.md diff --git a/site/versions/master/docs/install-windows.md b/site/docs/install-windows.md index 44713c291e..44713c291e 100644 --- a/site/versions/master/docs/install-windows.md +++ b/site/docs/install-windows.md diff --git a/site/docs/install.md b/site/docs/install.md index 1166fe133c..fe7c46e650 100644 --- a/site/docs/install.md +++ b/site/docs/install.md @@ -1,4 +1,87 @@ --- -layout: redirect -redirect: docs/install.html +layout: documentation +title: Installing Bazel --- + +# Installing Bazel + +See the instructions for installing Bazel on: + +* [Ubuntu Linux (16.04, 15.10, and 14.04)](install-ubuntu.md) +* [Mac OS X](install-os-x.md) +* [Windows (experimental)](install-windows.md) + +For other platforms, you can try to [compile from source](install-compile-source.md). + +Required Java version: + +Java JDK 8 or later is required. To address the problem of JDK 8 not being +available on some machines, Bazel's binary installer embeds a JDK by default. + +**Note:** Homebrew and Debian packages do not contain the embedded JDK. The +shell installers are the only ones with an embedded JDK. + +Extras: + +* [Bash completion](#bash) +* [zsh completion](#zsh) + +For more information on using Bazel, see +[Getting Started with Bazel](getting-started.html). + + +## <a name="jdk7"></a>Using Bazel with JDK 7 + +Bazel version _0.5.0_ does run with JDK 7. However, starting with version +_0.5.1_ Bazel must use JDK 8. + +The installers available for _0.5.0_ are: + +* `bazel-0.5.0-installer.sh`: default version with embedded JDK +* `bazel-0.5.0-without-jdk-installer.sh`: version without embedded JDK +* `bazel-0.5.0-jdk7-installer.sh`: version compatible with JDK 7, will not be + available in later releases + +## <a name="bash"></a>Getting bash completion + +Bazel comes with a bash completion script, which the installer copies into the +`bin` directory. If you ran the installer with `--user`, this will be +`$HOME/.bazel/bin`. If you ran the installer as root, this will be +`/usr/local/bazel/bin`. + +Copy the `bazel-complete.bash` script to your completion folder +(`/etc/bash_completion.d` directory under Ubuntu). If you don't have a +completion folder, you can copy it wherever suits you and insert +`source /path/to/bazel-complete.bash` in your `~/.bashrc` file (under OS X, put +it in your `~/.bash_profile` file). + +If you built Bazel from source, the bash completion target is in the `//scripts` +package: + +1. Build it with Bazel: `bazel build //scripts:bazel-complete.bash`. +2. Copy the script `bazel-bin/scripts/bazel-complete.bash` to one of the + locations described above. + +## <a name="zsh"></a>Getting zsh completion + +Bazel also comes with a zsh completion script. To install it: + +1. Add this script to a directory on your $fpath: + + ``` + fpath[1,0]=~/.zsh/completion/ + mkdir -p ~/.zsh/completion/ + cp scripts/zsh_completion/_bazel ~/.zsh/completion + ``` + + You may have to call `rm -f ~/.zcompdump; compinit` + the first time to make it work. + +2. Optionally, add the following to your .zshrc. + + ``` + # This way the completion script does not have to parse Bazel's options + # repeatedly. The directory in cache-path must be created manually. + zstyle ':completion:*' use-cache on + zstyle ':completion:*' cache-path ~/.zsh/cache + ``` diff --git a/site/docs/mobile-install.md b/site/docs/mobile-install.md index b209f182e5..f273871d0d 100644 --- a/site/docs/mobile-install.md +++ b/site/docs/mobile-install.md @@ -1,4 +1,220 @@ --- -layout: redirect -redirect: docs/mobile-install.html +layout: documentation +title: mobile-install --- + +# bazel mobile-install + +<p class="lead">Fast iterative development for Android</p> + +## TL;DR + +To install small changes to an Android app very quickly, do the following: + + 1. Find the `android_binary` rule of the app you want to install. + 2. Disable Proguard by removing the `proguard_specs` attribute. + 3. Set the `multidex` attribute to `native`. + 4. Set the `dex_shards` attribute to `10`. + 5. Connect your device running ART (not Dalvik) over USB and enable USB + debugging on it. + 6. Run `bazel mobile-install :your_target`. App startup will be a little + slower than usual. + 7. Edit the code or Android resources. + 8. Run `bazel mobile-install --incremental :your_target`. + 9. Enjoy not having to wait a lot. + +Some command line options to Bazel that may be useful: + + - `--adb` tells Bazel which adb binary to use + - `--adb_arg` can be used to add extra arguments to the command line of `adb`. + One useful application of this is to select which device you want to install + to if you have multiple devices connected to your workstation: + `bazel mobile-install --adb_arg=-s --adb_arg=<SERIAL> :your_target` + - `--start_app` automatically starts the app + +When in doubt, look at the +[example](https://github.com/bazelbuild/bazel/tree/master/examples/android) +or [contact us](https://groups.google.com/forum/#!forum/bazel-discuss). + +## Introduction + +One of the most important attributes of a developer's toolchain is speed: there +is a world of difference between changing the code and seeing it run within a +second and having to wait minutes, sometimes hours, before you get any feedback +on whether your changes do what you expect them to. + +Unfortunately, the traditional Android toolchain for building an .apk entails +many monolithic, sequential steps and all of these have to be done in order to +build an Android app. At Google, waiting five minutes to build a single-line +change was not unusual on larger projects like Google Maps. + +`bazel mobile-install` makes iterative development for Android much faster by +using a combination of change pruning, work sharding, and clever manipulation of +Android internals, all without changing any of your app's code. + +## Problems with traditional app installation + +We identified the following bottlenecks of building an Android app: + +- Dexing. By default, "dx" is invoked exactly once in the build and it does not +know how to reuse work from previous builds: it dexes every method again, even +though only one method was changed. + +- Uploading data to the device. adb does not use the full bandwidth of a USB 2.0 +connection, and larger apps can take a lot of time to upload. The entire app is +uploaded, even if only small parts have changed, for example, a resource or a +single method, so this can be a major bottleneck. + +- Compilation to native code. Android L introduced ART, a new Android runtime, +which compiles apps ahead-of-time rather than compiling them just-in-time like +Dalvik. This makes apps much faster at the cost of longer installation +time. This is a good tradeoff for users because they typically install an app +once and use it many times, but results in slower development where an app is +installed many times and each version is run at most a handful of times. + +## The approach of `bazel mobile-install` + +`bazel mobile-install `makes the following improvements: + + - Sharded dexing. After building the app's Java code, Bazel shards the class + files into approximately equal-sized parts and invokes `dx` separately on + them. `dx` is not invoked on shards that did not change since the last build. + + - Incremental file transfer. Android resources, .dex files, and native + libraries are removed from the main .apk and are stored in under a separate + mobile-install directory. This makes it possible to update code and Android + resources independently without reinstalling the whole app. Thus, + transferring the files takes less time and only the .dex files that have + changed are recompiled on-device. + + - Loading parts of the app from outside the .apk. A tiny stub application is + put into the .apk that loads Android resources, Java code and native code + from the on-device mobile-install directory, then transfers control to the + actual app. This is all transparent to the app, except in a few corner cases + described below. + +### Sharded Dexing + +Sharded dexing is reasonably straightforward: once the .jar files are built, a +[tool](https://github.com/bazelbuild/bazel/blob/master/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java) +shards them into separate .jar files of approximately equal size, then invokes +`dx` on those that were changed since the previous build. The logic that +determines which shards to dex is not specific to Android: it just uses the +general change pruning algorithm of Bazel. + +The first version of the sharding algorithm simply ordered the .class files +alphabetically, then cut the list up into equal-sized parts, but this proved to +be suboptimal: if a class was added or removed (even a nested or an anonymous +one), it would cause all the classes alphabetically after it to shift by one, +resulting in dexing those shards again. Thus, we settled upon sharding not +individual classes, but Java packages instead. Of course, this still results in +dexing many shards if a new package is added or removed, but that is much less +frequent than adding or removing a single class. + +The number of shards is controlled by the BUILD file (using the +`android_binary.dex_shards` attribute). In an ideal world, Bazel would +automatically determine how many shards are best, but Bazel currently must know +the set of actions (i.e. commands to be executed during the build) before +executing any of them, so it cannot determine the optimal number of shards +because it doesn't know how many Java classes there will eventually be in the +app. Generally speaking, the more shards, the faster the build and the +installation will be, but the slower app startup becomes, because the dynamic +linker has to do more work. The sweet spot is usually between 10 and 50 shards. + +### Incremental File Transfer + +After building the app, the next step is to install it, preferably with the +least effort possible. Installation consists of the following steps: + + 1. Installing the .apk (i.e. `adb install`) + 2. Uploading the .dex files, Android resources, and native libraries to the + mobile-install directory + +There is not much incrementality in the first step: the app is either installed +or not. Bazel currently relies on the user to indicate if it should do this step +through the `--incremental` command line option because it cannot determine in +all cases if it is necessary. + +In the second step, the app's files from the build are compared to an on-device +manifest file that lists which app files are on the device and their +checksums. Any new files are uploaded to the device, any files that have changed +are updated, and any files that have been removed are deleted from the +device. If the manifest is not present, it is assumed that every file needs to +be uploaded. + +Note that it is possible to fool the incremental installation algorithm by +changing a file on the device, but not its checksum in the manifest. We could +have safeguarded against this by computing the checksum of the files on the +device, but this was deemed to be not worth the increase in installation time. + +### The Stub Application + +The stub application is where the magic to load the dexes, native code and +Android resources from the on-device `mobile-install` directory happens. + +The actual loading is implemented by subclassing `BaseDexClassLoader` and is a +reasonably well-documented technique. This happens before any of the app's +classes are loaded, so that any application classes that are in the apk can be +placed in the on-device `mobile-install` directory so that they can be updated +without `adb install`. + +This needs to happen before any of the +classes of the app are loaded, so that no application class needs to be in the +.apk which would mean that changes to those classes would require a full +re-install. + +This is accomplished by replacing the `Application` class specified in +`AndroidManifest.xml` with the +[stub application](https://github.com/bazelbuild/bazel/blob/master/src/tools/android/java/com/google/devtools/build/android/incrementaldeployment/StubApplication.java). This +takes control when the app is started, and tweaks the class loader and the +resource manager appropriately at the earliest moment (its constructor) using +Java reflection on the internals of the Android framework. + +Another thing the stub application does is to copy the native libraries +installed by mobile-install to another location. This is necessary because the +dynamic linker needs the `X` bit to be set on the files, which is not possible to +do for any location accessible by a non-root `adb`. + +Once all these things are done, the stub application then instantiates the +actual `Application` class, changing all references to itself to the actual +application within the Android framework. + +## Results + +### Performance + +In general, `bazel mobile-install` results in a 4x to 10x speedup of building +and installing large apps after a small change. We computed the following +numbers for a few Google products: + +<img src="/assets/mobile-install-performance.svg"/> + +This, of course, depends on the nature of the change: recompilation after +changing a base library takes more time. + +### Limitations + +The tricks the stub application plays don't work in every case. We have +identified the following cases where it does not work as expected: + + - When `Context` is cast to the `Application` class in + `ContentProvider#onCreate()`. This method is called during application + startup before we have a chance to replace the instance of the `Application` + class, therefore, `ContentProvider` will still reference the stub application + instead of the real one. Arguably, this is not a bug since you are not + supposed to downcast `Context` like this, but this seems to happen in a few + apps at Google. + + - Resources installed by `bazel mobile-install` are only available from within + the app. If resources are accessed by other apps via + `PackageManager#getApplicationResources()`, these resources will be from the + last non-incremental install. + + - Devices that aren't running ART. While the stub application works well on + Froyo and later, Dalvik has a bug that makes it think that the app is + incorrect if its code is distributed over multiple .dex files in certain + cases, for example, when Java annotations are used in a + [specific](https://code.google.com/p/android/issues/detail?id=78144) way. As + long as your app doesn't tickle these bugs, it should work with Dalvik, too + (note, however, that support for old Android versions isn't exactly our + focus) diff --git a/site/docs/output_directories.md b/site/docs/output_directories.md index aa176bb43c..20858c6101 100644 --- a/site/docs/output_directories.md +++ b/site/docs/output_directories.md @@ -1,4 +1,131 @@ --- -layout: redirect -redirect: docs/output_directories.html +layout: documentation +title: Output Directory Layout --- + +# Output Directory Layout + +## Requirements + +Requirements for an output directory layout: + +* Don't collide if multiple users are building on the same box. +* Support building in multiple workspaces at the same time. +* Support building for multiple target configurations in the same workspace. +* Don't collide with any other tools. +* Be easy to access. +* Be easy to clean, even selectively. +* Is unambiguous, even if the user relies on symbolic links when changing into + his/her client directory. +* All the build state per user should be underneath one directory ("I'd like to + clean all the .o files from all my clients.") + +## Documentation of the current Bazel output directory layout + +The solution that's currently implemented: + +* Bazel must be invoked from a directory containing a WORKSPACE file. It reports + an error if it is not. We call this the _workspace directory_. +* The _outputRoot_ directory is ~/.cache/bazel. (Unless `$TEST_TMPDIR` is + set, as in a test of bazel itself, in which case this directory is used + instead.) +* We stick the Bazel user's build state beneath `outputRoot/_bazel_$USER`. This + is called the _outputUserRoot_ directory. +* Beneath the `outputUserRoot` directory, we create an `installBase` directory + whose name is "install" plus the MD5 hash of the Bazel installation manifest. +* Beneath the `outputUserRoot` directory, we also create an `outputBase` + directory whose name is the MD5 hash of the path name of the workspace + directory. So, for example, if Bazel is running in the workspace directory + `/home/user/src/my-project` (or in a directory symlinked to that one), then we + create an output base directory called: + `/home/.cache/bazel/_bazel_user/7ffd56a6e4cb724ea575aba15733d113`. +* Users can use Bazel's `--output_base` startup option to override the default + output base directory. For example, + `bazel --output_base=/tmp/bazel/output build x/y:z`. +* Users can also use Bazel's `--output_user_root` startup option to override the + default install base and output base directories. For example: + `bazel --output_user_root=/tmp/bazel build x/y:z`. + +We put symlinks "bazel-<workspace-name>" and "bazel-out", as well as +"bazel-bin", "bazel-genfiles", and "bazel-includes" in the workspace directory; +these symlinks points to some directories inside a target-specific directory +inside the output directory. These symlinks are only for the user's convenience, +as Bazel itself does not use them. Also, we only do this if the workspace +directory is writable. The names of the "bazel-bin", "bazel-genfiles", and +"bazel-include" symlinks are affected by the `--symlink_prefix` option to bazel, +but "bazel-<workspace-name>" and "bazel-out" are not. + +## Bazel internals: Directory layout + +The directories are laid out as follows: + +<pre> +<workspace-name>/ <== The workspace directory + bazel-my-project => <...my-project> <== Symlink to execRoot + bazel-out => <...bin> <== Convenience symlink to outputPath + bazel-bin => <...bin> <== Convenience symlink to most recent written bin dir $(BINDIR) + bazel-genfiles => <...genfiles> <== Convenience symlink to most recent written genfiles dir $(GENDIR) + +/home/user/.cache/bazel/ <== Root for all Bazel output on a machine: outputRoot + _bazel_$USER/ <== Top level directory for a given user depends on the user name: + outputUserRoot + install/ + fba9a2c87ee9589d72889caf082f1029/ <== Hash of the Bazel install manifest: installBase + _embedded_binaries/ <== Contains binaries and scripts unpacked from the data section of + the bazel executable on first run (e.g. helper scripts and the + main Java file BazelServer_deploy.jar) + 7ffd56a6e4cb724ea575aba15733d113/ <== Hash of the client's workspace directory (e.g. + /home/some-user/src/my-project): outputBase + action_cache/ <== Action cache directory hierarchy + This contains the persistent record of the file metadata + (timestamps, and perhaps eventually also MD5 sums) used by the + FilesystemValueChecker. + action_outs/ <== Action output directory. This contains a file with the + stdout/stderr for every action from the most recent bazel run + that produced output. + command.log <== A copy of the stdout/stderr output from the most recent bazel + command. + external/ <== The directory that remote repositories are downloaded/symlinked + into. + server/ <== The Bazel server puts all server-related files (such as socket + file, logs, etc) here. + jvm.out <== The debugging output for the server. + <workspace-name>/ <== Working tree for the Bazel build & root of symlink forest: execRoot + _bin/ <== Helper tools are linked from or copied to here. + + bazel-out/ <== All actual output of the build is under here: outputPath + local_linux-fastbuild/ <== one subdirectory per unique target BuildConfiguration instance; + this is currently encoded + bin/ <== Bazel outputs binaries for target configuration here: $(BINDIR) + foo/bar/_objs/baz/ <== Object files for a cc_* rule named //foo/bar:baz + foo/bar/baz1.o <== Object files from source //foo/bar:baz1.cc + other_package/other.o <== Object files from source //other_package:other.cc + foo/bar/baz <== foo/bar/baz might be the artifact generated by a cc_binary named + //foo/bar:baz + foo/bar/baz.runfiles/ <== The runfiles symlink farm for the //foo/bar:baz executable. + MANIFEST + <workspace-name>/ + ... + genfiles/ <== Bazel puts generated source for the target configuration here: + $(GENDIR) + foo/bar.h e.g. foo/bar.h might be a headerfile generated by //foo:bargen + testlogs/ <== Bazel internal test runner puts test log files here + foo/bartest.log e.g. foo/bar.log might be an output of the //foo:bartest test with + foo/bartest.status foo/bartest.status containing exit status of the test (e.g. + PASSED or FAILED (Exit 1), etc) + include/ <== a tree with include symlinks, generated as needed. The + bazel-include symlinks point to here. This is used for + linkstamp stuff, etc. + host/ <== BuildConfiguration for build host (user's workstation), for + building prerequisite tools, that will be used in later stages + of the build (ex: Protocol Compiler) + <packages>/ <== Packages referenced in the build appear as if under a regular workspace +</pre> + +The layout of the *.runfiles directories is documented in more detail in the places pointed to by RunfilesSupport. + +## `bazel clean` + +`bazel clean` does an `rm -rf` on the `outputPath` and the `action_cache` +directory. It also removes the workspace symlinks. The `--expunge` option +will clean the entire outputBase. diff --git a/site/docs/query-how-to.html b/site/docs/query-how-to.html deleted file mode 100644 index 26ba74e3c9..0000000000 --- a/site/docs/query-how-to.html +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/query-how-to.html ---- diff --git a/site/versions/master/docs/query-how-to.md b/site/docs/query-how-to.md index 98e8f87b48..98e8f87b48 100644 --- a/site/versions/master/docs/query-how-to.md +++ b/site/docs/query-how-to.md diff --git a/site/docs/query.md b/site/docs/query.md deleted file mode 100644 index 7a42ae7ea5..0000000000 --- a/site/docs/query.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/query.html ---- diff --git a/site/docs/rule-challenges.md b/site/docs/rule-challenges.md index 88aa5cba13..fece635b5a 100644 --- a/site/docs/rule-challenges.md +++ b/site/docs/rule-challenges.md @@ -1,4 +1,214 @@ --- -layout: redirect -redirect: docs/rule-challenges.html +layout: documentation +title: Challenges of Writing Rules. --- + +# Challenges of Writing Rules. + +We have heard feedback from various people that they have +difficulty to write efficient Bazel rules. There is no single root cause, but +it’s due to a combination of historical circumstances and intrinsic complexity +in the problem domain. This document attempts to give a high level overview of +the specific issues that we believe to be the main contributors. + +* Assumption: Aim for Correctness, Throughput, Ease of Use & Latency +* Assumption: Large Scale Repositories +* Assumption: BUILD-like Description Language +* Intrinsic: Remote Execution and Caching are Hard +* Historic: Hard Separation between Loading, Analysis, and Execution is + Outdated, but still affects the API +* Intrinsic: Using Change Information for Correct and Fast Incremental Builds + requires Unusual Coding Patterns +* Intrinsic: Avoiding Quadratic Time and Memory Consumption is Hard + +## Assumption: Aim for Correctness, Throughput, Ease of Use & Latency + +We assume that the build system needs to be first and foremost correct with +respect to incremental builds, i.e., for a given source tree, the output of the +same build should always be the same, regardless of what the output tree looks +like. In the first approximation, this means Bazel needs to know every single +input that goes into a given build step, such that it can rerun that step if any +of the inputs change. There are limits to how correct Bazel can get, as it leaks +some information such as date / time of the build, and ignores certain types of +changes such as changes to file attributes. Sandboxing helps ensure correctness +by preventing reads to undeclared input files. Besides the intrinsic limits of +the system, there are a few known correctness issues, most of which are related +to Fileset or the C++ rules, which are both hard problems. We have long-term +efforts to fix these. + +The second goal of the build system is to have high throughput; we are +permanently pushing the boundaries of what can be done within the current +machine allocation for a remote execution service. If the remote execution +service gets overloaded, nobody can get work done. + +Ease of use comes next, i.e., of multiple correct approaches with the same (or +similar) footprint of the remote execution service, we choose the one that is +easier to use. + +For the purpose of this document, latency denotes the time it takes from +starting a build to getting the intended result, whether that is a test log from +a passing or failing test, or an error message that a BUILD file has a +typo. + +Note that these goals often overlap; latency is as much a function of throughput +of the remote execution service as is correctness relevant for ease of use. + + +## Assumption: Large Scale Repositories + +The build system needs to operate at the scale of large repositories where large +scale means that it does not fit on a single hard drive, so it is impossible to +do a full checkout on virtually all developer machines. A medium-sized build +will need to read and parse tens of thousands of BUILD files, and evaluate +hundreds of thousands of globs. While it is theoretically possible to read all +BUILD files on a single machine, we have not yet been able to do so within a +reasonable amount of time and memory. As such, it is critical that BUILD files +can be loaded and parsed independently. + + +## Assumption: BUILD-like Description Language + +For the purpose of this document, we assume a configuration language that is +roughly similar to BUILD files, i.e., declaration of library and binary rules +and their interdependencies. BUILD files can be read and parsed independently, +and we avoid even looking at source files whenever we can (except for +existence). + + +## Intrinsic: Remote Execution and Caching are Hard + +Remote execution and caching improve build times in large repositories by +roughly two orders of magnitude compared to running the build on a single +machine. However, the scale at which it needs to perform is staggering: Google's +remote execution service is designed to handle a huge number of requests per +second, and the protocol carefully avoids unnecessary roundtrips as well as +unnecessary work on the service side. + +At this time, the protocol requires that the build system knows all inputs to a +given action ahead of time; the build system then computes a unique action +fingerprint, and asks the scheduler for a cache hit. If a cache hit is found, +the scheduler replies with the digests of the output files; the files itself are +addressed by digest later on. However, this imposes restrictions on the Bazel +rules, which need to declare all input files ahead of time. + + +## Historic: Hard Separation between Loading, Analysis, and Execution is Outdated, but still affects the API + +Technically, it is sufficient for a rule to know the input and output files of +an action just before the action is sent to remote execution. However, the +original Bazel code base had a strict separation of loading packages, then +analyzing rules using a configuration (command-line flags, essentially), and +only then running any actions. This distinction is still part of the rules API +today, even though the core of Bazel no longer requires it (more details below). + +That means that the rules API requires a declarative description of the rule +interface (what attributes it has, types of attributes). There are some +exceptions where the API allows custom code to run during the loading phase to +compute implicit names of output files and implicit values of attributes. For +example, a java_library rule named ‘foo’ implicitly generates an output named +‘libfoo.jar’, which can be referenced from other rules in the build graph. + +Furthermore, the analysis of a rule cannot read any source files or inspect the +output of an action; instead, it needs to generate a partial directed bipartite +graph of build steps and output file names that is only determined from the rule +itself and its dependencies. + + +## Intrinsic: Using Change Information for Correct and Fast Incremental Builds requires Unusual Coding Patterns + +Above, we argued that in order to be correct, Bazel needs to know all the input +files that go into a build step in order to detect whether that build step is +still up-to-date. The same is true for package loading and rule analysis, and we +have designed [Skyframe] (http://www.bazel.build/docs/skyframe.html) to handle this +in general. Skyframe is a graph library and evaluation framework that takes a +goal node (such as ‘build //foo with these options’), and breaks it down into +its constituent parts, which are then evaluated and combined to yield this +result. As part of this process, Skyframe reads packages, analyzes rules, and +executes actions. + +At each node, Skyframe tracks exactly which nodes any given node used to compute +its own output, all the way from the goal node down to the input files (which +are also Skyframe nodes). Having this graph explicitly represented in memory +allows the build system to identify exactly which nodes are affected by a given +change to an input file (including creation or deletion of an input file), doing +the minimal amount of work to restore the output tree to its intended state. + +As part of this, each node performs a dependency discovery process; i.e., each +node can declare dependencies, and then use the contents of those dependencies +to declare even further dependencies. In principle, this maps well to a +thread-per-node model. However, medium-sized builds contain hundreds of +thousands of Skyframe nodes, which isn’t easily possible with current Java +technology (and for historical reasons, we’re currently tied to using Java, so +no lightweight threads and no continuations). + +Instead, Bazel uses a fixed-size thread pool. However, that means that if a node +declares a dependency that isn’t available yet, we may have to abort that +evaluation and restart it (possibly in another thread), when the dependency is +available. This, in turn, means that nodes should not do this excessively; a +node that declares N dependencies serially can potentially be restarted N times, +costing O(N^2) time. Instead, we aim for up-front bulk declaration of +dependencies, which sometimes requires reorganizing the code, or even splitting +a node into multiple nodes to limit the number of restarts. + +Note that this technology isn’t currently available in the rules API; instead, +the rules API is still defined using the legacy concepts of loading, analysis, +and execution phases. However, a fundamental restriction is that all accesses to +other nodes have to go through the framework so that it can track the +corresponding dependencies. Regardless of the language in which the build system +is implemented or in which the rules are written (they don’t have to be the +same), rule authors must not use standard libraries or patterns that bypass +Skyframe. For Java, that means avoiding java.io.File as well as any form of +reflection, and any library that does either. Libraries that support dependency +injection of these low-level interfaces still need to be setup correctly for +Skyframe. + +This strongly suggests to avoid exposing rule authors to a full language runtime +in the first place. The danger of accidental use of such APIs is just too big - +several Bazel bugs in the past were caused by rules using unsafe APIs, even +though the rules were written by the Bazel team, i.e., by the domain experts. + + +## Intrinsic: Avoiding Quadratic Time and Memory Consumption is Hard + +To make matters worse, apart from the requirements imposed by Skyframe, the +historical constraints of using Java, and the outdatedness of the rules API, +accidentally introducing quadratic time or memory consumption is a fundamental +problem in any build system based on library and binary rules. There are two +very common patterns that introduce quadratic memory consumption (and therefore +quadratic time consumption). + +1. Chains of Library Rules - +Consider the case of a chain of library rules A depends on B, depends on C, and +so on. Then, we want to compute some property over the transitive closure of +these rules, such as the Java runtime classpath, or the C++ linker command for +each library. Naively, we might take a standard list implementation; however, +this already introduces quadratic memory consumption: the first library +contains one entry on the classpath, the second two, the third three, and so +on, for a total of 1+2+3+...+N = O(N^2) entries. + +2. Binary Rules Depending on the Same Library Rules - +Consider the case where a set of binaries that depend on the same library +rules; for example, you might have a number of test rules that test the same +library code. Let’s say out of N rules, half the rules are binary rules, and +the other half library rules. Now consider that each binary makes a copy of +some property computed over the transitive closure of library rules, such as +the Java runtime classpath, or the C++ linker command line. For example, it +could expand the command line string representation of the C++ link action. N/2 +copies of N/2 elements is O(N^2) memory. + + +### Custom Collections Classes to Avoid Quadratic Complexity + +Bazel is heavily affected by both of these scenarios, so we introduced a set of +custom collection classes that effectively compress the information in memory by +avoiding the copy at each step. Almost all of these data structures have set +semantics, so we called the class NestedSet. The majority of changes to reduce +Bazel’s memory consumption over the past several years were changes to use +NestedSet instead of whatever was previously used. + +Unfortunately, usage of NestedSet does not automatically solve all the issues; +in particular, even just iterating over a NestedSet in each rule re-introduces +quadratic time consumption. NestedSet also has some helper methods to facilitate +interoperability with normal collections classes; unfortunately, accidentally +passing a NestedSet to one of these methods leads to copying behavior, and +reintroduces quadratic memory consumption. diff --git a/site/docs/skyframe.md b/site/docs/skyframe.md deleted file mode 100644 index d0835fd632..0000000000 --- a/site/docs/skyframe.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: regular-redirect -redirect: /designs/skyframe.html ----
\ No newline at end of file diff --git a/site/docs/skylark/aspects.md b/site/docs/skylark/aspects.md index ba524f5cf6..5dcaf1bedf 100644 --- a/site/docs/skylark/aspects.md +++ b/site/docs/skylark/aspects.md @@ -1,4 +1,191 @@ --- -layout: redirect -redirect: docs/skylark/aspects.html +layout: documentation +title: Aspects --- +# Aspects + +**Status: Experimental**. We may make breaking changes to the API, but we will + help you update your code. + +Aspects allow augmenting build dependency graphs with additional information +and actions. Some typical scenarios when aspects can be useful: + +* IDEs that integrate Bazel can use aspects to collect information about the + project +* Code generation tools can leverage aspects to execute on their inputs in + "target-agnostic" manner. As an example, BUILD files can specify a hierarchy + of [protobuf](https://developers.google.com/protocol-buffers/) library + definitions, and language-specific rules can use aspects to attach + actions generating protobuf support code for a particular language + +## Aspect basics + +Bazel BUILD files provide a description of a project’s source code: what source +files are part of the project, what artifacts (_targets_) should be built from +those files, what the dependencies between those files are, etc. Bazel uses +this information to perform a build, that is, it figures out the set of actions +needed to produce the artifacts (such as running compiler or linker) and +executes those actions. Bazel accomplishes this by constructing a _dependency +graph_ between targets and visiting this graph to collect those actions. + +Consider the following BUILD file: + +```python +java_library(name = 'W', ...) +java_library(name = 'Y', deps = [':W'], ...) +java_library(name = 'Z', deps = [':W'], ...) +java_library(name = 'Q', ...) +java_library(name = 'T', deps = [':Q'], ...) +java_library(name = 'X', deps = [':Y',':Z'], runtime_deps = [':T'], ...) +``` + +This BUILD file defines a dependency graph shown in Fig 1. + +<img src="build-graph.png" alt="Build Graph" width="250px" /> + +Bazel analyzes this dependency graph by calling implementations of +[rules](rules.md) (in this case "java_library" starting from leaves of +the dependency graph). These implementations generate actions that build +artifacts (such as Jar files), and provide information (such as locations +and names of those artifacts) to their dependencies in providers that +they return. Their dependencies can access those providers through the +[Target object](lib/Target.html). In other words, every target +defined in the BUILD file generates a node in the dependency graph, and +the appropriate rule implementation function is called for every node. + +Aspects are similar to rules in that they have an implementation function that +generates actions and returns providers. However, their power comes from +the way the dependency graph is built for them. An aspect has an implementation +and a list of all attributes it propagates along. Consider an aspect A that +propagates along attributes named "deps". This aspect can be applied to +a target X, yielding an aspect application node A(X). During its application, +aspect A is applied recursively to all targets that X refers to in its "deps" +attribute (all attributes in A's propagation list). Thus a single act of +applying aspect A to a target X yields a "shadow graph" of the original +dependency graph of targets (see Fig.2). + + + +The only edges that are shadowed are the edges along the attributes in +the propagation set, thus the `runtime_deps` edge is not shadowed in this +example. An aspect implementation function is then invoked on all nodes in +the shadow graph similar to how rule implementations are invoked on the nodes +of the original graph. + +## Defining aspects + +Aspect definitions are similiar to rule definitions. Let's take a look at +the example: + +```python +metal_proto_aspect = aspect(implementation = _metal_proto_aspect_impl, + attr_aspects = ["deps"], + attrs = { + "_protoc" : attr.label( + default=Label("//tools:metal_protoc"), + executable = True + ) + } +) +``` + +Just like a rule, an aspect has an implementation function. ``attr_aspects`` +specify the aspect's propagation set: a list of attributes of rules along which +the aspect propagates. + +``attrs`` defines a set of attributes for aspects. Aspects are only allowed +to have private attributes of types ``label`` or ``label_list``. Attributes +can be used to specify dependencies on tools or libraries that are needed +for actions generated by aspects. + +### Implementation functions + +Aspect implementation functions are similiar to the rule implementation +functions. They return [providers](rules.md#providers), can generate +[actions](rules.md#actions) and take two arguments: + +* `target`: the [target](lib/Target.html) the aspect is being applied to. +* `ctx`: [`ctx`](lib/ctx.html) object that can be used to access attributes and + generate outputs and actions. + +Example: + +```python +def _metal_proto_aspect_impl(target, ctx): + # For every `src` in proto_library, generate an output file + proto_sources = [f for src in ctx.rule.attr.srcs + for f in src.files] + outputs = [ctx.new_file(f.short_path + ".metal") + for f in proto_sources] + ctx.action( + executable = ctx.executable._protoc, + argument = ... + inputs = proto_sources + outputs = outputs) + transitive_outputs = depset(outputs) + for dep in ctx.rule.attr.deps: + transitive_outputs = transitive_outputs | dep.metal_proto.transitive_outputs + return struct( + metal_proto = struct(direct_outputs = outputs, + transitive_outputs = transitive_outputs)) +``` + +The implementation function can access the attributes of the target rule via +[`ctx.rule.attr`](lib/ctx.html#rule). It can examine providers that are +provided by the target to which it is applied (via the `target` argument). + +Just like a rule implementation function, an aspect implementation function +returns a struct of providers that are accessible to its dependencies. + +* The set of providers for an aspect application A(X) is the union of providers + that come from the implementation of a rule for target X and from + the implementation of aspect A. It is an error if a target and an aspect that + is applied to it each provide a provider with the same name. +* For the aspect implementation, the values of attributes along which + the aspect is propagated (from the `attr_aspect` list) are replaced with + the results of an application of the aspect to them. For example, if target + X has Y and Z in its deps, `ctx.rule.attr.deps` for A(X) will be [A(Y), A(Z)]. + In the `_metal_proto_aspect_impl` function above, ctx.rule.attr.deps will be + Target objects that are the results of applying the aspect to the 'deps' + of the original target to which the aspect has been applied. + That allows the aspect to examine `metal_proto` provider on them. + + +## Applying aspects + +Aspect propagation can be initiated either from a rule or from the command line. + +### Applying aspects to rule attributes + +Rules can specify that they want to apply aspects to their dependencies. +The aspects to be applied to a particular attribute can be specified +using the `aspects` parameter to `attr.label` or `attr.label_list` function: + +```python +metal_proto_library = rule(implementation = _impl, + attrs = { + 'proto_deps' : attr.label_list(aspects = [metal_proto_aspect]), + }, +) +``` + +If a rule specifies an aspect on its attributes, the values of that attribute +will be replaced by the result of aspect application to them (similar to +what happens during aspect propagation). Thus implementation of +`metal_proto_library` will have access to `metal_proto` providers +on the target objects representing its `proto_deps` attribute values. + +### Applying aspects from command line. + +Aspects can also be applied on the command line, using the `--aspects` flag: + + +``` +bazel build //java/com/company/example:main \ + --aspects path/to/extension.bzl%metal_proto_aspect +``` + +`--aspects` flag takes one argument, which is a specification of the aspect in +the format `<extension file path>%<aspect top-level name>`. + + diff --git a/site/versions/master/docs/skylark/build-graph-aspect.svg b/site/docs/skylark/build-graph-aspect.svg index 508a916bd3..508a916bd3 100644 --- a/site/versions/master/docs/skylark/build-graph-aspect.svg +++ b/site/docs/skylark/build-graph-aspect.svg diff --git a/site/versions/master/docs/skylark/build-graph-aspects.png b/site/docs/skylark/build-graph-aspects.png Binary files differindex fb8042a5b7..fb8042a5b7 100644 --- a/site/versions/master/docs/skylark/build-graph-aspects.png +++ b/site/docs/skylark/build-graph-aspects.png diff --git a/site/versions/master/docs/skylark/build-graph.png b/site/docs/skylark/build-graph.png Binary files differindex 2efd68ac59..2efd68ac59 100644 --- a/site/versions/master/docs/skylark/build-graph.png +++ b/site/docs/skylark/build-graph.png diff --git a/site/versions/master/docs/skylark/build-graph.svg b/site/docs/skylark/build-graph.svg index cbad06dcb2..cbad06dcb2 100644 --- a/site/versions/master/docs/skylark/build-graph.svg +++ b/site/docs/skylark/build-graph.svg diff --git a/site/docs/skylark/build-style.md b/site/docs/skylark/build-style.md index 5a47a30eda..d7787593aa 100644 --- a/site/docs/skylark/build-style.md +++ b/site/docs/skylark/build-style.md @@ -1,4 +1,114 @@ --- -layout: redirect -redirect: docs/skylark/build-style.html +layout: documentation +title: Style guide for BUILD files --- + +# BUILD file style guide + +In `BUILD` files, we take the same approach as in Go: We let the machine take care +of most formatting issues. +[Buildifier](https://github.com/bazelbuild/buildifier) is a tool that parses and +emits the source code in a standard style. Every `BUILD` file is therefore +formatted in the same automated way, which makes formatting a non-issue during +code reviews. It also makes it easier for tools to understand, edit, and +generate `BUILD` files. + +`BUILD` file formatting must match the output of `buildifier`. + +## Formatting example + +```python +package(default_visibility = ["//visibility:public"]) + +py_test( + name = "foo_test", + srcs = glob(["*.py"]), + data = [ + "//data/production/foo:startfoo", + "//foo", + "//third_party/java/jdk:jdk-k8", + ], + flaky = 1, + deps = [ + ":check_bar_lib", + ":foo_data_check", + ":pick_foo_port", + "//pyglib", + "//testing/pybase", + ], +) +``` + +## File structure + +We recommend to use the following order (every element is optional): + + * Package description (a comment) + + * All `load()` statements + + * The `package()` function. + + * Calls to rules and macros + +Buildifier makes a distinction between a standalone comment and a comment +attached to an element. If a comment is not attached to a specific element, use +an empty line after it. The distinction is important when doing automated +changes (e.g. to decide if we keep or remove a comment when we delete a rule). + +```python +# Standalone comment (e.g. to make a section in a file) + +# Comment for the cc_library below +cc_library(name = "cc") +``` + +## Conventions + + * Use uppercase and underscores to declare constants (e.g. `GLOBAL_CONSTANT`), + use lowercase and underscores to declare variables (e.g. `my_variable`). + + * Labels should be canonicalized. Use `//foo/bar` instead of `//foo/bar:bar`. + Use `:bar` if it is defined in the same package. *Rationale*: It makes clear + if a label is local to a package. Sorting a list of labels is messy if all + labels do not use the same conventions. + + * Labels should never be split, even if they are longer than 79 characters. + Labels should be string literals whenever possible. Rationale: It makes + find and replace easy. It also improves readability. + + * The value of the name attribute should be a literal constant string (except + in macros). *Rationale*: External tools use the name attribute to refer a + rule. They need to find rules without having to interpret code. + +## Differences with Python style guide + +Although compatibility with +[Python style guide](https://www.python.org/dev/peps/pep-0008/) is a goal, there +are a few differences: + + * No strict line length limit. Long comments and long strings are often split + to 79 columns, but it is not required. It should not be enforced in code + reviews or presubmit scripts. *Rationale*: Labels can be long and exceed this + limit. It is common for `BUILD` files to be generated or edited by tools, which + does not go well with a line length limit. + + * Implicit string concatenation is not supported. Use the `+` operator. + *Rationale*: `BUILD` files contain many string lists. It is easy to forget a + comma, which leads to a complete different result. This has created many bugs + in the past. [See also this discussion.](https://lwn.net/Articles/551438/) + + * Use spaces around the `=` sign for keywords arguments in rules. *Rationale*: + Named arguments are much more frequent than in Python and are always on a + separate line. Spaces improve readability. This convention has been around + for a long time, and we don't think it is worth modifying all existing + `BUILD` files. + + * By default, use double quotation marks for strings. *Rationale*: This is not + specified in the Python style guide, but it recommends consistency. So we + decided to use only double-quoted strings. Many languages use double-quotes + for string literals. + + * Use a single blank line between two top-level definitions. *Rationale*: The + structure of a `BUILD` file is not like a typical Python file. It has only + top-level statements. Using a single-blank line makes `BUILD` files shorter. diff --git a/site/docs/skylark/bzl-style.md b/site/docs/skylark/bzl-style.md index c17600796b..66c2d2918e 100644 --- a/site/docs/skylark/bzl-style.md +++ b/site/docs/skylark/bzl-style.md @@ -1,4 +1,52 @@ --- -layout: redirect -redirect: docs/skylark/bzl-style.html +layout: documentation +title: Style guide for bzl files --- + +# .bzl file style guide + +## Style + +* When in doubt, follow the + [Python style guide](https://www.python.org/dev/peps/pep-0008/). + +* Code should be documented using + [docstrings](https://www.python.org/dev/peps/pep-0257/). Use a docstring at + the top of the file, and a docstring for each public function. + +* Variables and function names use lowercase with words separated by underscores + (`[a-z][a-z0-9_]*`), e.g. `cc_library`. Top-level private values start with + one underscore. Bazel enforces that private values cannot be used from other + files. + +* As in BUILD files, there is no strict line length limit as labels can be long. + When possible, try to use at most 79 characters per line. + +* In keyword arguments, spaces around the equal sign are optional. In general, + we follow the BUILD file convention when calling macros and native rules, and + the Python convention for other functions, e.g. + +```python +def fct(name, srcs): + filtered_srcs = my_filter(source=srcs) + native.cc_library( + name = name, + srcs = filtered_srcs, + ) +``` + +## Macros + +A [macro](macros.md) is a function which instantiates one or many rules during +the loading phase. + +* Macros must accept a name attribute and each invocation should specify a name. + The generated name attribute of rules should include the name attribute as a + prefix. For example, `my_macro(name = "foo")` can generate a rule `foo` and a + rule `foo_gen`. *Rationale*: Users should be able to find easily which macro + generated a rule. Also, automated refactoring tools need a way to identify a + specific rule to edit. + +* When calling a macro, use only keyword arguments. *Rationale*: This is for + consistency with rules, it greatly improves readability. + diff --git a/site/docs/skylark/concepts.md b/site/docs/skylark/concepts.md index a1e5028f15..8caf553dfb 100644 --- a/site/docs/skylark/concepts.md +++ b/site/docs/skylark/concepts.md @@ -1,4 +1,220 @@ --- -layout: redirect -redirect: docs/skylark/concepts.html +layout: documentation +title: Extensions - Overview --- +# Overview + + +## Loading an extension + +Extensions are files with the `.bzl` extension. Use the `load` statement to +import a symbol from an extension. + +```python +load("//build_tools/rules:maprule.bzl", "maprule") +``` + +This code will load the file `build_tools/rules/maprule.bzl` and add the +`maprule` symbol to the environment. This can be used to load new rules, +functions or constants (e.g. a string, a list, etc.). Multiple symbols can be +imported by using additional arguments to the call to `load`. Arguments must +be string literals (no variable) and `load` statements must appear at +top-level, i.e. they cannot be in a function body. + +`load` also supports aliases, i.e. you can assign different names to the +imported symbols. + +```python +load("//build_tools/rules:maprule.bzl", maprule_alias = "maprule") +``` + +You can define multiple aliases within one `load` statement. Moreover, the +argument list can contain both aliases and regular symbol names. The following +example is perfectly legal (please note when to use quotation marks). + +```python +load(":my_rules.bzl", "some_rule", nice_alias = "some_other_rule") +``` + +In a `.bzl` file, symbols starting with `_` are private and cannot be loaded +from another file. Visibility doesn't affect loading (yet): you don't need to +use `exports_files` to make a `.bzl` file visible. + +## Macros and rules + +A [macro](macros.md) is a function that instantiates rules. It is useful when a +BUILD file is getting too repetitive or too complex, as it allows you to reuse +some code. The function is evaluated as soon as the BUILD file is read. After +the evaluation of the BUILD file, Bazel has little information about macros: if +your macro generates a `genrule`, Bazel will behave as if you wrote the +`genrule`. As a result, `bazel query` will only list the generated `genrule`. + +A [rule](rules.md) is more powerful than a macro. It can access Bazel internals +and have full control over what is going on. It may for example pass information +to other rules. + +If you want to reuse simple logic, start with a macro. If a macro becomes +complex, it is often a good idea to make it a rule. Support for a new language +is typically done with a rule. Rules are for advanced users: we expect that most +people will never have to write one, they will only load and call existing +rules. + +## Evaluation model + +A build consists of three phases. + +* **Loading phase**. First, we load and evaluate all extensions and all BUILD + files that are needed for the build. The execution of the BUILD files simply + instantiates rules (each time a rule is called, it gets added to a graph). + This is where macros are evaluated. + +* **Analysis phase**. The code of the rules is executed (their `implementation` + function), and actions are instantiated. An action describes how to generate + a set of outputs from a set of inputs, e.g. "run gcc on hello.c and get + hello.o". It is important to note that we have to list explicitly which + files will be generated before executing the actual commands. In other words, + the analysis phase takes the graph generated by the loading phase and + generates an action graph. + +* **Execution phase**. Actions are executed, when at least one of their outputs is + required. If a file is missing or if a command fails to generate one output, + the build fails. Tests are also run during this phase. + +Bazel uses parallelism to read, parse and evaluate the `.bzl` files and `BUILD` +files. A file is read at most once per build and the result of the evaluation is +cached and reused. A file is evaluated only once all its dependencies (`load()` +statements) have been resolved. By design, loading a `.bzl` file has no visible +side-effect, it only defines values and functions. + +Bazel tries to be clever: it uses dependency analysis to know which files must +be loaded, which rules must be analyzed, and which actions must be executed. For +example, if a rule generates actions that we don't need for the current build, +they will not be executed. + +## Backward-incompatible changes + +As we make changes and polish the extension mechanism, old features may be +removed and new features that are not backwards-compatible may be added. + +Each release, new incompatible changes will be behind a flag with its default +value set to `false`. In later releases, the flag will be enabled by default, or +the flag will be removed entirely. + +To check if your code will be compatible with future releases: + +* build your code with the flag `--all_incompatible_changes`, or +* use boolean flags to enable/disable specific incompatible changes. + +This following are the planned incompatible changes that are implemented and +guarded behind flags. + +### Set constructor + +We are removing the `set` constructor. Use `depset` instead. `set` and `depset` +are equivalent, you just need to do search and replace to update the old code. + +We are doing this to reduce confusion between the specialized +[depset](depsets.md) data structure and Python's set datatype. + +* Flag: `--incompatible_disallow_set_constructor=true` +* Introduced in: 0.5.1 + +### Keyword-only arguments + +Keyword-only parameters are parameters that can be called only using their name. + +``` python +def foo(arg1, *, arg2): pass + +foo(3, arg2=3) +``` + +``` python +def bar(arg1, *rest, arg2): pass + +bar(3, arg2=3) +``` + +In both examples, `arg2` must be named at the call site. To preserve syntactic +compatibility with Python 2, we are removing this feature (which we have never +documented). + +* Flag: `--incompatible_disallow_keyword_only_args=true` +* Introduced in: 0.5.1 + +### Mutating `+=` + +We are changing `left += right` when `left` is a list. The old behavior is +equivalent to `left = left + right`, which creates a new list and assigns it to +`left`. The new behavior does not rebind `left`, but instead just mutates the +list in-place. + +``` python +def fct(): + li = [1] + alias = li + li += [2] + # Old behavior: alias == [1] + # New behavior: alias == [1, 2] +``` + +This change makes Skylark more compatible with Python and avoids performance +issues. The `+=` operator for tuples is unaffected. + +* Flag: `--incompatible_list_plus_equals_inplace=true` +* Introduced in: 0.5.1 + +### Dictionary concatenation + +We are removing the `+` operator on dictionaries. This includes the `+=` form +where the left-hand side is a dictionary. This is done to improve compatibility +with Python. A possible workaround is to use the `.update` method instead. + +* Flag: `--incompatible_disallow_dict_plus=true` +* Introduced in: 0.5.1 + +## Upcoming changes + +The following items are upcoming changes. + +* Comprehensions currently "leak" the values of their loop variables into the + surrounding scope (Python 2 semantics). This will be changed so that + comprehension variables are local (Python 3 semantics). + +* Previously dictionaries were guaranteed to use sorted order for their keys. + Going forward, there is no guarantee on order besides that it is + deterministic. As an implementation matter, some kinds of dictionaries may + continue to use sorted order while others may use insertion order. + +These changes concern the `load()` syntax in particular. + +* Currently a `load()` statement can appear anywhere in a file so long as it is + at the top-level (not in an indented block of code). In the future they will + be required to appear at the beginning of the file, i.e., before any + non-`load()` statement. + +* In BUILD files, `load()` can overwrite an existing variable with the loaded + symbol. This will be disallowed in order to improve consistency with .bzl + files. Use load aliases to avoid name clashes. + +* The .bzl file can be specified as either a path or a label. In the future only + the label form will be allowed. + +* Cross-package visibility restrictions do not yet apply to loaded .bzl files. + At some point this will change. In order to load a .bzl from another package + it will need to be exported, such as by using an `exports_files` declaration. + The exact syntax has not yet been decided. + + +## Profiling the code + +To profile your code and analyze the performance, use the `--profile` flag: + +``` +$ bazel build --nobuild --profile=/tmp/prof //path/to:target +$ bazel analyze-profile /tmp/prof --html --html_details +``` + +Then, open the generated HTML file (`/tmp/prof.html` in the example). + + diff --git a/site/docs/skylark/cookbook.md b/site/docs/skylark/cookbook.md index a4a4dadcbb..28303e2457 100644 --- a/site/docs/skylark/cookbook.md +++ b/site/docs/skylark/cookbook.md @@ -1,4 +1,950 @@ --- -layout: redirect -redirect: docs/skylark/cookbook.html +layout: documentation +title: Extensions examples --- +# Extensions examples + +## <a name="macro"></a>Macro creating a rule + +An example of a macro creating a rule. + +`empty.bzl`: + +```python +def _impl(ctx): + print("This rule does nothing") + +empty = rule(implementation=_impl) +``` + +`extension.bzl`: + +```python +# Loading the rule. The rule doesn't have to be in a separate file. +load("//pkg:empty.bzl", "empty") + +def macro(name, visibility=None): + # Creating the rule. + empty(name = name, visibility = visibility) +``` + +`BUILD`: + +```python +load("//pkg:extension.bzl", "macro") + +macro(name = "myrule") +``` + +## <a name="macro_native"></a>Macro creating a native rule + +An example of a macro creating a native rule. Native rules are special rules +that are automatically available (without <code>load</code>). They are +accessed using the <a href="lib/native.html">native</a> module. + +`extension.bzl`: + +```python +def macro(name, visibility=None): + # Creating a native genrule. + native.genrule( + name = name, + outs = [name + '.txt'], + cmd = 'echo hello > $@', + visibility = visibility, + ) +``` + +`BUILD`: + +```python +load("//pkg:extension.bzl", "macro") + +macro(name = "myrule") +``` + +## <a name="macro_compound"></a>Macro multiple rules + +There's currently no easy way to create a rule that directly uses the +action of a native rule. You can work around this using macros: + +```python +def _impl(ctx): + return struct([...], + # When instrumenting this rule, again hide implementation from + # users. + instrumented_files( + source_attributes = ["srcs", "csrcs"], + dependency_attributes = ["deps", "cdeps"])) + +# This rule is private and can only be accessed from the current file. +_cc_and_something_else_binary = rule(implementation=_impl) + + +# This macro is public, it's the public interface to instantiate the rule. +def cc_and_something_else_binary(name, srcs, deps, csrcs, cdeps): + cc_binary_name = "%s.cc_binary" % name + + native.cc_binary( + name = cc_binary_name, + srcs = csrcs, + deps = cdeps, + visibility = ["//visibility:private"] + ) + + _cc_and_something_else_binary( + name = name, + srcs = srcs, + deps = deps, + # A label attribute so that this depends on the internal rule. + cc_binary = cc_binary_name, + # Redundant labels attributes so that the rule with this target name knows + # about everything it would know about if cc_and_something_else_binary + # were an actual rule instead of a macro. + csrcs = csrcs, + cdeps = cdeps) +``` + + +## <a name="conditional-instantiation"></a>Conditional instantiation + +Macros can look at previously instantiated rules. This is done with +`native.existing_rule`, which returns information on a single rule defined in the same +`BUILD` file, eg., + +```python +native.existing_rule("descriptor_proto") +``` + +This is useful to avoid instantiating the same rule twice, which is an +error. For example, the following macro will simulate a test suite, +instantiating tests for diverse flavors of the same test. + +`extension.bzl`: + +```python +def system_test(name, test_file, flavor): + n = "system_test_%s_%s_test" % (test_file, flavor) + if native.existing_rule(n) == None: + native.py_test( + name = n, + srcs = [ + "test_driver.py", + test_file, + ], + args = ["--flavor=" + flavor], + ) + return n + +def system_test_suite(name, flavors=["default"], test_files=[]): + ts = [] + for flavor in flavors: + for test in test_files: + ts.append(system_test(name, test, flavor)) + native.test_suite(name = name, tests = ts) +``` + +In the following BUILD file, note how `(basic_test.py, fast)` is emitted for +both the `smoke` test suite and the `thorough` test suite. + +`BUILD`: + +```python +load("//pkg:extension.bzl", "system_test_suite") + +# Run all files through the 'fast' flavor. +system_test_suite( + name = "smoke", + flavors = ["fast"], + test_files = glob(["*_test.py"]), +) + +# Run the basic test through all flavors. +system_test_suite( + name = "thorough", + flavors = [ + "fast", + "debug", + "opt", + ], + test_files = ["basic_test.py"], +) +``` + + +## <a name="aggregation"></a>Aggregating over the BUILD file + +Macros can collect information from the BUILD file as processed so far. We call +this aggregation. The typical example is collecting data from all rules of a +certain kind. This is done by calling +<a href="lib/native.html#existing_rules">native.existing\_rules</a>, which +returns a dictionary representing all rules defined so far in the current BUILD +file. The dictionary has entries of the form `name` => `rule`, with the values +using the same format as `native.existing_rule`. + +```python +def archive_cc_src_files(tag): + """Create an archive of all C++ sources that have the given tag.""" + all_src = [] + for r in native.existing_rules().values(): + if tag in r["tags"] and r["kind"] == "cc_library": + all_src.append(r["srcs"]) + native.genrule(cmd = "zip $@ $^", srcs = all_src, outs = ["out.zip"]) +``` + +Since `native.existing_rules` constructs a potentially large dictionary, you should avoid +calling it repeatedly within BUILD file. + +## <a name="empty"></a>Empty rule + +Minimalist example of a rule that does nothing. If you build it, the target will +succeed (with no generated file). + +`empty.bzl`: + +```python +def _impl(ctx): + # You may use print for debugging. + print("This rule does nothing") + +empty = rule(implementation=_impl) +``` + +`BUILD`: + +```python +load("//pkg:empty.bzl", "empty") + +empty(name = "nothing") +``` + +## <a name="attr"></a>Rule with attributes + +Example of a rule that shows how to declare attributes and access them. + +`printer.bzl`: + +```python +def _impl(ctx): + # You may use print for debugging. + print("Rule name = %s, package = %s" % (ctx.label.name, ctx.label.package)) + + # This prints the labels of the deps attribute. + print("There are %d deps" % len(ctx.attr.deps)) + for i in ctx.attr.deps: + print("- %s" % i.label) + # A label can represent any number of files (possibly 0). + print(" files = %s" % [f.path for f in i.files]) + +printer = rule( + implementation=_impl, + attrs={ + # Do not declare "name": It is added automatically. + "number": attr.int(default = 1), + "deps": attr.label_list(allow_files=True), + }) +``` + +`BUILD`: + +```python +load("//pkg:printer.bzl", "printer") + +printer( + name = "nothing", + deps = [ + "BUILD", + ":other", + ], +) + +printer(name = "other") +``` + +If you execute this file, some information is printed as a warning by the +rule. No file is generated. + +## <a name="shell"></a>Simple shell command + +Example of a rule that runs a shell command on an input file specified by +the user. The output has the same name as the rule, with a `.size` suffix. + +While convenient, Shell commands should be used carefully. Generating the +command-line can lead to escaping and injection issues. It can also create +portability problems. It is often better to declare a binary target in a +BUILD file and execute it. See the example [executing a binary](#execute-bin). + +`size.bzl`: + +```python +def _impl(ctx): + output = ctx.outputs.out + input = ctx.file.file + # The command may only access files declared in inputs. + ctx.action( + inputs=[input], + outputs=[output], + progress_message="Getting size of %s" % input.short_path, + command="stat -L -c%%s %s > %s" % (input.path, output.path)) + +size = rule( + implementation=_impl, + attrs={"file": attr.label(mandatory=True, allow_files=True, single_file=True)}, + outputs={"out": "%{name}.size"}, +) +``` + +`foo.txt`: + +``` +Hello +``` + +`BUILD`: + +```python +load("//pkg:size.bzl", "size") + +size( + name = "foo_size", + file = "foo.txt", +) +``` + +## <a name="file"></a>Write string to a file + +Example of a rule that writes a string to a file. + +`file.bzl`: + +```python +def _impl(ctx): + output = ctx.outputs.out + ctx.file_action(output=output, content=ctx.attr.content) + +file = rule( + implementation=_impl, + attrs={"content": attr.string()}, + outputs={"out": "%{name}.txt"}, +) +``` + +`BUILD`: + +```python +load("//pkg:file.bzl", "file") + +file( + name = "hello", + content = "Hello world", +) +``` + + +## <a name="execute-bin"></a>Execute a binary + +This rule executes an existing binary. In this particular example, the +binary is a tool that merges files. During the analysis phase, we cannot +access any arbitrary label: the dependency must have been previously +declared. To do so, the rule needs a label attribute. In this example, we +will give the label a default value and make it private (so that it is not +visible to end users). Keeping the label private can simplify maintenance, +since you can easily change the arguments and flags you pass to the tool. + +`execute.bzl`: + +```python +def _impl(ctx): + # The list of arguments we pass to the script. + args = [ctx.outputs.out.path] + [f.path for f in ctx.files.srcs] + # Action to call the script. + ctx.action( + inputs=ctx.files.srcs, + outputs=[ctx.outputs.out], + arguments=args, + progress_message="Merging into %s" % ctx.outputs.out.short_path, + executable=ctx.executable._merge_tool) + +concat = rule( + implementation=_impl, + attrs={ + "srcs": attr.label_list(allow_files=True), + "out": attr.output(mandatory=True), + "_merge_tool": attr.label(executable=True, cfg="host", allow_files=True, + default=Label("//pkg:merge")) + } +) +``` + +Any executable target can be used. In this example, we will use a +`sh_binary` rule that concatenates all the inputs. + +`BUILD`: + +``` +load("execute", "concat") + +concat( + name = "sh", + srcs = [ + "header.html", + "body.html", + "footer.html", + ], + out = "page.html", +) + +# This target is used by the shell rule. +sh_binary( + name = "merge", + srcs = ["merge.sh"], +) +``` + +`merge.sh`: + +```python +#!/bin/bash + +out=$1 +shift +cat $* > $out +``` + +`header.html`: + +``` +<html><body> +``` + +`body.html`: + +``` +content +``` + +`footer.html`: + +``` +</body></html> +``` + +## <a name="execute"></a>Execute an input binary + +This rule has a mandatory `binary` attribute. It is a label that can refer +only to executable rules or files. + +`execute.bzl`: + +```python +def _impl(ctx): + # ctx.new_file is used for temporary files. + # If it should be visible for user, declare it in rule.outputs instead. + f = ctx.new_file(ctx.configuration.bin_dir, "hello") + # As with outputs, each time you declare a file, + # you need an action to generate it. + ctx.file_action(output=f, content=ctx.attr.input_content) + + ctx.action( + inputs=[f], + outputs=[ctx.outputs.out], + executable=ctx.executable.binary, + progress_message="Executing %s" % ctx.executable.binary.short_path, + arguments=[ + f.path, + ctx.outputs.out.path, # Access the output file using + # ctx.outputs.<attribute name> + ] + ) + +execute = rule( + implementation=_impl, + attrs={ + "binary": attr.label(cfg="host", mandatory=True, allow_files=True, + executable=True), + "input_content": attr.string(), + "out": attr.output(mandatory=True), + }, +) +``` + +`a.sh`: + +```bash +#!/bin/bash + +tr 'a-z' 'A-Z' < $1 > $2 +``` + +`BUILD`: + +```python +load("//pkg:execute.bzl", "execute") + +execute( + name = "e", + input_content = "some text", + binary = "a.sh", + out = "foo", +) +``` + +## <a name="runfiles"></a>Runfiles and location substitution + +`execute.bzl`: + +```python +def _impl(ctx): + executable = ctx.outputs.executable + command = ctx.attr.command + # Expand the label in the command string to a runfiles-relative path. + # The second arg is the list of labels that may be expanded. + command = ctx.expand_location(command, ctx.attr.data) + # Create the output executable file with command as its content. + ctx.file_action( + output=executable, + content=command, + executable=True) + + return [DefaultInfo( + # Create runfiles from the files specified in the data attribute. + # The shell executable - the output of this rule - can use them at runtime. + # It is also possible to define data_runfiles and default_runfiles. + # However if runfiles is specified it's not possible to define the above + # ones since runfiles sets them both. + # Remember, that the struct returned by the implementation function needs + # to have a field named "runfiles" in order to create the actual runfiles + # symlink tree. + runfiles=ctx.runfiles(files=ctx.files.data) + )] + +execute = rule( + implementation=_impl, + executable=True, + attrs={ + "command": attr.string(), + "data": attr.label_list(cfg="data", allow_files=True), + }, +) +``` + +`data.txt`: + +``` +Hello World! +``` + +`BUILD`: + +```python +load("//pkg:execute.bzl", "execute") + +execute( + name = "e", + # The location will be expanded to "pkg/data.txt", and it will reference + # the data.txt file in runfiles when this target is invoked as + # "bazel run //pkg:e". + command = "cat $(location :data.txt)", + data = [':data.txt'] +) +``` + +## <a name="late-bound"></a>Computed dependencies + +Bazel needs to know about all dependencies before doing the analysis phase and +calling the implementation function. Dependencies can be computed based on the +rule attributes: to do so, use a function as the default +value of an attribute (the attribute must be private and have type `label` or +`list of labels`). The parameters of this function must correspond to the +attributes that are accessed in the function body. + +Note: For legacy reasons, the function takes the configuration as an additional +parameter. Please do not rely on the configuration since it will be removed in +the future. + +The example below computes the md5 sum of a file. The file can be preprocessed +using a filter. The exact dependencies depend on the filter chosen by the user. + +`hash.bzl`: + +```python +_filters = { + "comments": Label("//pkg:comments"), + "spaces": Label("//pkg:spaces"), + "none": None, +} + +def _get_filter(filter, cfg=None): # requires attribute "filter" + # Return the value for the attribute "_filter_bin" + # It can be a label or None. + return _filters[filter] + +def _impl(ctx): + src = ctx.file.src + + if not ctx.attr._filter_bin: + # Skip the processing + processed = src + else: + processed = ctx.new_file(ctx.label.name + "_processed") + # Run the selected binary + ctx.action( + outputs = [processed], + inputs = [ctx.file.src], + progress_message="Apply filter '%s'" % ctx.attr.filter, + arguments = [ctx.file.src.path, processed.path], + executable = ctx.executable._filter_bin) + + # Compute the hash + out = ctx.outputs.text + ctx.action( + outputs = [out], + inputs = [processed], + command = "md5sum < %s > %s" % (processed.path, out.path)) + +md5_sum = rule( + implementation=_impl, + attrs={ + "filter": attr.string(values=_filters.keys(), default="none"), + "src": attr.label(mandatory=True, single_file=True, allow_files=True), + "_filter_bin": attr.label(default=_get_filter, executable=True), + }, + outputs = {"text": "%{name}.txt"}) +``` + +`BUILD`: + +```python +load("//pkg:hash.bzl", "md5_sum") + +md5_sum( + name = "hash", + src = "hello.txt", + filter = "spaces", +) + +sh_binary( + name = "comments", + srcs = ["comments.sh"], +) + +sh_binary( + name = "spaces", + srcs = ["spaces.sh"], +) +``` + +`hello.txt`: + +``` +Hello World! +``` + +`comments.sh`: + +``` +#!/bin/bash +grep -v '^ *#' $1 > $2 # Remove lines with only a Python-style comment +``` + +`spaces.sh`: + +``` +#!/bin/bash +tr -d ' ' < $1 > $2 # Remove spaces +``` + +## <a name="mandatory-providers"></a>Mandatory providers + +In this example, rules have a `number` attribute. Each rule adds its +number with the numbers of its transitive dependencies, and write the +result in a file. This shows how to transfer information from a dependency +to its dependents. + +`sum.bzl`: + +```python +NumberInfo = provider() + +def _impl(ctx): + result = ctx.attr.number + for dep in ctx.attr.deps: + result += dep[NumberInfo].number + ctx.file_action(output=ctx.outputs.out, content=str(result)) + + # Return the provider with result, visible to other rules. + return [NumberInfo(number=result)] + +sum = rule( + implementation=_impl, + attrs={ + "number": attr.int(default=1), + # All deps must provide all listed providers. + "deps": attr.label_list(providers=[NumberInfo]), + }, + outputs = {"out": "%{name}.sum"} +) +``` + +`BUILD`: + +```python +load("//pkg:sum.bzl", "sum") + +sum( + name = "n", + deps = ["n2", "n5"], +) + +sum( + name = "n2", + number = 2, +) + +sum( + name = "n5", + number = 5, +) +``` + +## <a name="optional-providers"></a>Optional providers + +This is a similar example, but dependencies may not provide a number. + +`sum.bzl`: + +```python +NumberInfo = provider() + +def _impl(ctx): + result = ctx.attr.number + for dep in ctx.attr.deps: + if NumberInfo in dep: + result += dep[NumberInfo].number + ctx.file_action(output=ctx.outputs.out, content=str(result)) + + # Return the provider with result, visible to other rules. + return [NumberInfo(number=result)] + +sum = rule( + implementation=_impl, + attrs={ + "number": attr.int(default=1), + "deps": attr.label_list(), + }, + outputs = {"out": "%{name}.sum"} +) +``` + +`BUILD`: + +```python +load("//pkg:sum.bzl", "sum") + +sum( + name = "n", + deps = ["n2", "n5"], +) + +sum( + name = "n2", + number = 2, +) + +sum( + name = "n5", + number = 5, +) +``` + +## <a name="outputs-executable"></a>Default executable output + +This example shows how to create a default executable output. + +`extension.bzl`: + +```python +def _impl(ctx): + ctx.file_action( + # Access the executable output file using ctx.outputs.executable. + output=ctx.outputs.executable, + content="#!/bin/bash\necho Hello!", + executable=True + ) + # The executable output is added automatically to this target. + +executable_rule = rule( + implementation=_impl, + executable=True +) +``` + +`BUILD`: + +```python +load("//pkg:extension.bzl", "executable_rule") + +executable_rule(name = "my_rule") +``` + +## <a name="outputs-default"></a>Default outputs + +This example shows how to create default outputs for a rule. + +`extension.bzl`: + +```python +def _impl(ctx): + ctx.file_action( + # Access the default outputs using ctx.outputs.<output name>. + output=ctx.outputs.my_output, + content="Hello World!" + ) + # The default outputs are added automatically to this target. + +rule_with_outputs = rule( + implementation=_impl, + outputs = { + # %{name} is substituted with the rule's name + "my_output": "%{name}.txt" + } +) +``` + +`BUILD`: + +```python +load("//pkg:extension.bzl", "rule_with_outputs") + +rule_with_outputs(name = "my_rule") +``` + +## <a name="outputs-custom"></a>Custom outputs + +This example shows how to create custom (user defined) outputs for a rule. +This rule takes a list of output file name templates from the user and +creates each of them containing a "Hello World!" message. + +`extension.bzl`: + +```python +def _impl(ctx): + # Access the custom outputs using ctx.outputs.<attribute name>. + for output in ctx.outputs.outs: + ctx.file_action( + output=output, + content="Hello World!" + ) + # The custom outputs are added automatically to this target. + +rule_with_outputs = rule( + implementation=_impl, + attrs={ + "outs": attr.output_list() + } +) +``` + +`BUILD`: + +```python +load("//pkg:extension.bzl", "rule_with_outputs") + +rule_with_outputs( + name = "my_rule", + outs = ["my_output.txt"] +) +``` + +## <a name="master-rule"></a>Master rules + +This example shows how to create master rules to bind other rules together. The +code below uses genrules for simplicity, but this technique is more useful with +other rules. For example, if you need to compile C++ files, you can reuse +`cc_library`. + +`extension.bzl`: + +```python +def _impl(ctx): + # Aggregate the output files from the depending rules + files = depset() + files += ctx.attr.dep_rule_1.files + files += ctx.attr.dep_rule_2.files + return [DefaultInfo(files=files)] + +# This rule binds the depending rules together +master_rule = rule( + implementation=_impl, + attrs={ + "dep_rule_1": attr.label(), + "dep_rule_2": attr.label() + } +) + +def macro(name, cmd, input): + # Create the depending rules + name_1 = name + "_dep_1" + name_2 = name + "_dep_2" + native.genrule( + name = name_1, + cmd = cmd, + outs = [name_1 + ".txt"] + ) + native.genrule( + name = name_2, + cmd = "echo " + input + " >$@", + outs = [name_2 + ".txt"] + ) + # Create the master rule + master_rule( + name = name, + dep_rule_1 = ":" + name_1, + dep_rule_2 = ":" + name_2 + ) +``` + +`BUILD`: + +```python +load("//pkg:extension.bzl", "macro") + +# This creates the target :my_rule +macro( + name = "my_rule", + cmd = "echo something > $@", + input = "Hello World" +) +``` + +## <a name="debugging-tips"></a>Debugging tips + +Here are some examples on how to debug macros and rules using +<a href="lib/globals.html#print">print</a>. + +`debug.bzl`: + +```python +print("print something when the module is loaded") + +def _impl(ctx): + print("print something when the rule implementation is executed") + print(type("abc")) # prints string, the type of "abc" + print(dir(ctx)) # prints all the fields and methods of ctx + print(dir(ctx.attr)) # prints all the attributes of the rule + # prints the objects each separated with new line + print("object1", "object2", sep="\n") + +debug = rule(implementation=_impl) +``` + +`BUILD`: + +```python +load("//pkg:debug.bzl", "debug") + +debug( + name = "printing_rule" +) +``` + diff --git a/site/docs/skylark/deploying.md b/site/docs/skylark/deploying.md index a56f7183ae..658445175e 100644 --- a/site/docs/skylark/deploying.md +++ b/site/docs/skylark/deploying.md @@ -1,4 +1,144 @@ --- -layout: redirect -redirect: docs/skylark/deploying.html +layout: documentation +title: Deploying new Skylark rules --- +# Deploying new Skylark rules + +This documentation is for Skylark rule writers who are planning to make their +rules available to others. + +## Where to put new rules + +In general, new rules should go into their own GitHub repository under your +organization. Contact the [bazel-dev mailing +list](https://groups.google.com/forum/#!forum/bazel-dev) if you feel like your +rules belong in the bazelbuild organization. + +You can see lots of examples of what your repository should look like on GitHub: +see all of the repositories named `rules_whatever`. In particular, +[rules_scala](https://github.com/bazelbuild/rules_scala) is a nice example of +how to set up your repo. + +Rules can be grouped either by language (e.g., Scala) or some notion of platform +(e.g., Android). + +## What a rule repository should contain + +Every rule repository should have a certain layout so that users can quickly +understand new rules. + +For example, suppose we are writing new Skylark rules for the (make-believe) +chaiscript language. We would have the following structure: + +``` +.travis.yml +README.md +WORKSPACE +chaiscript/ + BUILD + chaiscript.bzl +tests/ + BUILD + some_test.sh + another_test.py +examples/ + BUILD + bin.chai + lib.chai + test.chai +``` + +### README.md + +At the top level, there should be a README.md that contains (at least) what +users will need to copy-paste into their WORKSPACE file to use your rule. +In general, this will be a `git_repository` pointing to your GitHub repo and +a macro call that downloads/configures any tools your rule needs. For example, +for the [Go +rules](https://github.com/bazelbuild/rules_go/blob/master/README.md#setup), this +looks like: + +``` +git_repository( + name = "io_bazel_rules_go", + remote = "https://github.com/bazelbuild/rules_go.git", + tag = "0.0.2", +) +load("@io_bazel_rules_go//go:def.bzl", "go_repositories") + +go_repositories() +``` + +If your rules depend on another repository's rules, specify both in the +`README.md` (see the [Skydoc rules](https://github.com/bazelbuild/skydoc#setup), +which depend on the Sass rules, for an example of this). + +### Tests + +There should be tests that verify that the rules are working as expected. This +can either be in the standard location for the language the rules are for or a +`tests/` directory at the top level. + +### Optional: Examples + +It is useful to users to have an `examples/` directory that shows users a couple +of basic ways that the rules can be used. + +## Testing + +Set up Travis as described in their [getting started +docs](https://docs.travis-ci.com/user/getting-started/). Then add a +`.travis.yml` file to your repository with the following content: + +``` +language: + - java +jdk: + - oraclejdk8 # Building Bazel requires JDK8. +before_install: + - wget https://github.com/bazelbuild/bazel/archive/0.3.0.zip # Replace with desired version + - unzip 0.3.0.zip + - cd bazel-0.3.0 + - ./compile.sh + - sudo cp output/bazel /usr/bin/bazel + - cd .. + - rm -rf bazel-0.3.0 +script: + - bazel build //... + - bazel test //... +``` + +Right now Bazel has to be compiled from source, as Travis does not support a +version of GCC that works with the precompiled Bazel binaries. Thus, the +`before_install` steps download the Bazel source, compile it, and "install" the +Bazel binary in /usr/bin. + +If your repository is under the [bazelbuild organization](https://github.com/bazelbuild), +contact the [bazel-dev](https://groups.google.com/forum/#!forum/bazel-dev) list +to have it added to [ci.bazel.build](http://ci.bazel.build). + +## Documentation + +See the [Skydoc documentation](https://github.com/bazelbuild/skydoc) for +instructions on how to comment your rules so that documentation can be generated +automatically. + +## FAQs + +### Why can't we add our rule to the Bazel GitHub repository? + +We want to decouple rules from Bazel releases as much as possible. It's clearer +who owns individual rules, reducing the load on Bazel developers. For our users, +decoupling makes it easier to modify, upgrade, downgrade, and replace rules. +Contributing to rules can be lighter weight than contributing to Bazel - +depending on the rules -, including full submit access to the corresponding +GitHub repository. Getting submit access to Bazel itself is a much more involved +process. + +The downside is a more complicated one-time installation process for our users: +they have to copy-paste a rule into their WORKSPACE file, as shown in the +README section above. + +We used to have all of the Skylark rules in the Bazel repository (under +`//tools/build_rules` or `//tools/build_defs`). We still have a couple rules +there, but we are working on moving the remaining rules out. diff --git a/site/versions/master/docs/skylark/depsets.md b/site/docs/skylark/depsets.md index 24114b3200..24114b3200 100644 --- a/site/versions/master/docs/skylark/depsets.md +++ b/site/docs/skylark/depsets.md diff --git a/site/versions/master/docs/skylark/errors/read-only-variable.md b/site/docs/skylark/errors/read-only-variable.md index 5dc1eeebab..5dc1eeebab 100644 --- a/site/versions/master/docs/skylark/errors/read-only-variable.md +++ b/site/docs/skylark/errors/read-only-variable.md diff --git a/site/docs/skylark/index.md b/site/docs/skylark/index.md index 541fcba640..dc336c99da 100644 --- a/site/docs/skylark/index.md +++ b/site/docs/skylark/index.md @@ -1,4 +1,17 @@ --- -layout: redirect -redirect: docs/skylark/index.html +layout: documentation +title: Extensions --- + +# Extensions +Skylark is the name of the extension mechanism in Bazel. It lets you add support +for new languages and tools by writing [custom build rules](rules.md). You can +also compose existing rules into [macros](macros.md). + +## Getting started + +Read the [concepts](concepts.md) behind Skylark and try the +[cookbook examples](cookbook.md). To go further, read about the +[standard library](lib/globals.html). +--> + diff --git a/site/versions/master/docs/skylark/language.md b/site/docs/skylark/language.md index 583e4f7c0a..583e4f7c0a 100644 --- a/site/versions/master/docs/skylark/language.md +++ b/site/docs/skylark/language.md diff --git a/site/docs/skylark/macros.md b/site/docs/skylark/macros.md index b247e4b89f..7782198287 100644 --- a/site/docs/skylark/macros.md +++ b/site/docs/skylark/macros.md @@ -1,4 +1,156 @@ --- -layout: redirect -redirect: docs/skylark/macros.html +layout: documentation +title: Macros --- +# Macros + +## Macro creation + +A macro is a function called from the BUILD file that can instantiate rules. +Macros don't give additional power, they are just used for encapsulation and +code reuse. By the end of the [loading phase](concepts.md#evaluation-model), +macros don't exist anymore, and Bazel sees only the set of rules they created. + +Native rules (i.e. rules that don't need a `load()` statement) can be +instantiated from the [native](lib/native.html) module, e.g. + +```python +def my_macro(name, visibility=None): + native.cc_library( + name = name, + srcs = ["main.cc"], + visibility = visibility, + ) +``` + +If you need to know the package name (i.e. which BUILD file is calling the +macro), use the constant [PACKAGE_NAME](lib/globals.html#PACKAGE_NAME). + +## Examples + +* [Macro creating rules](cookbook.md#macro). + +* [Macro creating native rules](cookbook.md#macro_native). + +* [Macro combining multiple rules](cookbook.md#macro_compound). + +## Debugging + +* `bazel query --output=build //my/path:all` will show you how the BUILD file + looks after evaluation. All macros, globs, loops are expanded. Known + limitation: `select` expressions are currently not shown in the output. + +* You may filter the output based on `generator_function` (which function + generated the rules) or `generator_name` (the name attribute of the macro), + e.g. + ```bash + $ bazel query --output=build 'attr(generator_function, my_macro, //my/path:all)' + ``` + +* To find out where exactly the rule `foo` is generated in a BUILD file, you + can try the following trick. Insert this line near the top of the BUILD + file: `cc_library(name = "foo")`. Run Bazel. You will get an exception when + the rule `foo` is created (due to a name conflict), which will show you the + full stack trace. + +* You can also use [print](lib/globals.html#print) for debugging. It displays + the message as a warning during the loading phase. Except in rare cases, + either remove `print` calls, or make them conditional under a `debugging` + parameter that defaults to `False` before submitting the code to the depot. + +## Errors + +If you want to throw an error, use the [fail](lib/globals.html#fail) function. +Explain clearly to the user what went wrong and how to fix their BUILD file. It +is not possible to catch an error. + +``` +def my_macro(name, deps, visibility=None): + if len(deps) < 2: + fail("Expected at least two values in deps") + # ... +``` + +## Conventions + +* All public functions (functions that don't start with underscore) that + instantiate rules must have a `name` argument. This argument should not be + optional (don't give a default value). + +* Public functions should use a docstring following [Python + conventions](https://www.python.org/dev/peps/pep-0257/#one-line-docstrings). + +* In BUILD files, the `name` argument of the macros must be a keyword argument + (not a positional argument). + +* The `name` attribute of rules generated by a macro should include the name + argument as a prefix. For example, `macro(name = "foo")` can generate a + `cc_library` `foo` and a genrule `foo_gen`. + +* In most cases, optional parameters should have a default value of `None`. + `None` can be passed directly to native rules, which treat it the same as if + you had not passed in any argument. Thus, there is no need to replace it + with `0`, `False`, or `[]` for this purpose. Instead, the macro should defer + to the rules it creates, as their defaults may be complex or may change over + time. Additionally, a parameter that is explicitly set to its default value + looks different than one that is never set (or set to `None`) when accessed + through the query language or build-system internals. + +* Macros should have an optional `visibility` argument. + +## Full example + +The typical use-case for a macro is when you want to reuse a genrule, e.g. + +``` +genrule( + name = "file", + outs = ["file.txt"], + cmd = "$(location generator) some_arg > $@", + tools = [":generator"], +) +``` + +If you want to generate another file with different arguments, you may want to +extract this code to a function. + +The BUILD file will become simply: + +``` +load("//path:generator.bzl", "file_generator") + +file_generator( + name = "file", + arg = "some_arg", +) +``` + +In order to keep BUILD files clean and declarative, you must put the function in +a separate `.bzl` file. For example, write the definition of the macro in +`path/generator.bzl`: + +``` +def file_generator(name, arg, visibility=None): + native.genrule( + name = name, + outs = [name + ".txt"], + cmd = "$(location generator) %s > $@" % arg, + tools = ["//test:generator"], + visibility = visibility, + ) +``` + +When you want to investigate what a macro does, use the following command to +see the expanded form: + +``` +$ bazel query --output=build :file +# /absolute/path/test/ext.bzl:42:3 +genrule( + name = "file", + tools = ["//test:generator"], + outs = ["//test:file.txt"], + cmd = "$(location generator) some_arg > $@", +) +``` + diff --git a/site/docs/skylark/repository_rules.md b/site/docs/skylark/repository_rules.md index 6af0109578..c6890062ef 100644 --- a/site/docs/skylark/repository_rules.md +++ b/site/docs/skylark/repository_rules.md @@ -1,4 +1,113 @@ --- -layout: redirect -redirect: docs/skylark/repository_rules.html +layout: documentation +title: Repository Rules --- +# Repository Rules + +**Status: Experimental**. We may make breaking changes to the API, but we will + announce them and help you update your code. + +An [external repository](/docs/external.md) is a rule that can be used only +in the `WORKSPACE` file and enable non-hermetic operation at the loading phase +of Bazel. Each external repository rule creates its own workspace, with its +own BUILD files and artifacts. They can be used to depend on third-party +libraries (such as Maven packaged libraries) but also to generate BUILD files +specific to the host Bazel is running on. + +## Repository Rule creation + +In a `.bzl` file, use the +[repository_rule](lib/globals.html#repository_rule) function to create a new +repository rule and store it in a global variable. + +A custom repository rule can be used just like a native repository rule. It +has a mandatory `name` attribute and every target present in its build files +can be referred as `@<name>//package:target` where `<name>` is the value of the +`name` attribute. + +The rule is loaded when you explicitly build it, or if it is a dependency of +the build. In this case, Bazel will execute its `implementation` function. This +function describe how to creates the repository, its content and BUILD files. + +## Attributes + +An attribute is a rule argument, such as `url` or `sha256`. You must list +the attributes and their types when you define a repository rule. + +```python +local_repository = repository_rule( + implementation=_impl, + local=True, + attrs={"path": attr.string(mandatory=True)}) +``` + +`name` attributes are implicitly defined for all `repository_rule`s. +To access an attribute, use `repository_ctx.attr.<attribute_name>`. +The name of a repository rule is accessible with `repository_ctx.name`. + +If an attribute name starts with `_` it is private and users cannot set it. + +## Implementation function + +Every repository rule requires an `implementation` function. It contains the +actual logic of the rule and is executed strictly in the Loading Phase. +The function has exactly one input parameter, `repository_ctx`, and should +always returns `None`. The input parameter `repository_ctx` can be used to +access attribute values, and non-hermetic functions (finding a binary, +executing a binary, creating a file in the repository or downloading a file +from the Internet). See [the library](lib/repository_ctx.html) for more +context. Example: + +```python +def _impl(repository_ctx): + repository_ctx.symlink(repository_ctx.attr.path, "") + +local_repository = repository_rule( + implementation=_impl, + ...) +``` + +## When is the implementation function executed? + +If the repository is declared as `local` then change in a dependency +in the dependency graph (including the WORKSPACE file itself) will +cause an execution of the implementation function. + +The implementation function can be _restarted_ if a dependency it +request is _missing_. The beginning of the implementation function +will be re-executed after the dependency has been resolved. + +File given as a label are declared as dependencies, so requesting it +might interrupt the function and restart it later, re-executing the +part up till there. + +Finally, for non-`local` repositories, only a change in the following +dependencies might cause a restart: + +- Skylark files needed to define the repository rule. +- Declaration of the repository rule in the `WORKSPACE` file. +- Value of any environment variable declared with the `environ` +attribute of the +[`repository_rule`](https://bazel.build/versions/master/docs/skylark/lib/globals.html#repository_rule) +function. The value of those environment variable can be enforced from +the command line with the +[`--action_env`](/docs/command-line-reference.html#flag--action_env) +flag (but this flag will invalidate every action of the build). +- Content of any file used and referred to by a label (e.g., + `//mypkg:label.txt` not `mypkg/label.txt`). + +## Examples + +- [C++ auto-configured toolchain](https://github.com/bazelbuild/bazel/blob/ac29b78000afdb95afc7e97efd2b1299ebea4dac/tools/cpp/cc_configure.bzl#L288): +it uses a repository rule to automatically create the +C++ configuration files for Bazel by looking for the local C++ compiler, the +environment and the flags the C++ compiler supports. +- + [Go repositories](https://github.com/bazelbuild/rules_go/blob/67bc217b6210a0922d76d252472b87e9a6118fdf/go/private/go_repositories.bzl#L195) + uses several `repository_rule` to defines the list of dependencies + needed to use the Go rules. +- + [maven_jar](https://github.com/bazelbuild/bazel/a110ac400190c90a45856f15482c8d0952c542f5/master/tools/build_defs/repo/maven_rules.bzl#L276) + is a reimplementation of the native `maven_jar` rule using the + `maven` tool. + diff --git a/site/docs/skylark/rules.md b/site/docs/skylark/rules.md index 22b62057eb..47f27abb13 100644 --- a/site/docs/skylark/rules.md +++ b/site/docs/skylark/rules.md @@ -1,4 +1,596 @@ --- -layout: redirect -redirect: docs/skylark/rules.html +layout: documentation +title: Rules --- +# Rules + +**Status: Experimental**. We may make breaking changes to the API, but we will + help you update your code. + +A rule defines a series of [actions](#actions) that Bazel should perform on +inputs to get a set of outputs. For example, a C++ binary rule might take a set +of `.cpp` files (the inputs), run `g++` on them (the action), and return an +executable file (the output). + +Note that, from Bazel's perspective, `g++` and the standard C++ libraries are +also inputs to this rule. As a rule writer, you must consider not only the +user-provided inputs to a rule, but also all of the tools and libraries required +to execute the actions (called _implicit inputs_). + +Before creating or modifying any rule, make sure you are familiar with the +[extensibility model](concepts.md) (understand the three phases and the +differences between macros and rules). + +## Rule creation + +In a `.bzl` file, use the [rule](lib/globals.html#rule) +function to create a new rule and store it in a global variable: + +```python +my_rule = rule(...) +``` + +See [the cookbook](cookbook.md#empty) for examples. The rule can then be +loaded by BUILD files: + +```python +load('//some/pkg:whatever.bzl', 'my_rule') +``` + +A custom rule can be used just like a native rule. It has a mandatory `name` +attribute, you can refer to it with a label, and you can see it in +`bazel query`. + +The rule is analyzed when you explicitly build it, or if it is a dependency of +the build. In this case, Bazel will execute its `implementation` function. This +function decides what the outputs of the rule are and how to build them (using +[actions](#actions)). During the [analysis phase](concepts.md#evaluation-model), +no external command can be executed. Instead, actions are registered and +will be run in the execution phase, if their output is needed for the build. + +## Attributes + +An attribute is a rule argument, such as `srcs` or `deps`. You must list +the attributes and their types when you define a rule. + +```python +sum = rule( + implementation = _impl, + attrs = { + "number": attr.int(default = 1), + "deps": attr.label_list(), + }, +) +``` + +The following attributes are implicitly added to every rule: `deprecation`, +`features`, `name`, `tags`, `testonly`, `visibility`. Test rules also have the +following attributes: `args`, `flaky`, `local`, `shard_count`, `size`, +`timeout`. + +Labels listed in `attr` will be inputs to the rule. + +To access an attribute in a rule's implementation, use +`ctx.attr.<attribute_name>`. The name and the package of a rule are available +with `ctx.label.name` and `ctx.label.package`. + +See [an example](cookbook.md#attr) of using `attr` in a rule. + +### <a name="private-attributes"></a> Private Attributes + +In Python, we use one leading underscore(`_`) for non-public methods and +instance variables (see [PEP-8][1]). + +Similarly, if an attribute name starts with `_` it is private and users cannot +set it. +It is useful in particular for label attributes (your rule will have an +implicit dependency on this label). + +```python +metal_compile = rule( + implementation = _impl, + attrs = { + "srcs": attr.label_list(), + "_compiler": attr.label( + default = Label("//tools:metalc"), + allow_single_file = True, + executable = True, + ), + }, +) +``` + +## Implementation function + +Every rule requires an `implementation` function. It contains the actual logic +of the rule and is executed strictly in the +[analysis phase](concepts.md#evaluation-model). The function has exactly one +input parameter, `ctx`, and it may return the [runfiles](#runfiles) +and [providers](#providers) of the rule. The input parameter `ctx` can be used +to access attribute values, outputs and dependent targets, and files. It also +has some helper functions. See [the library](lib/ctx.html) for more context. +Example: + +```python +def _impl(ctx): + ... + return [DefaultInfo(runfiles=...), MyInfo(...)] + +my_rule = rule( + implementation = _impl, + ... +) +``` + +## Files + +There are two kinds of files: files stored in the file system and generated +files. For each generated file, there must be one and only one generating +action, and each action must generate one or more output files. Bazel will throw +an error otherwise. + +## Targets + +Every build rule corresponds to exactly one target. A target can create +[actions](#actions), can have dependencies (which can be files or +other build rules), [output files](#output-files) (generated by +its actions), and [providers](#providers). + +A target `y` depends on target `x` if `y` has a label or label list type +attribute where `x` is declared: + +```python +my_rule( + name = "x", +) + +my_rule( + name = "y", + deps = [":x"], +) +``` + +In the above case, it's possible to access targets declared in `my_rule.deps`: + +```python +def _impl(ctx): + for dep in ctx.attr.deps: + # Do something with dep + ... + +my_rule = rule( + implementation = _impl, + attrs = { + "deps": attr.label_list(), + }, + ... +) +``` + +## <a name="output-files"></a> Output files + +A target can declare output files, which must be generated by the target's +actions. There are three ways to create output files: + +* If the rule is marked `executable`, it creates an output file of the same name + as the rule's. [See example](cookbook.md#outputs-executable) + +* The rule can declare default `outputs`, which are always generated. + [See example](cookbook.md#outputs-default) + +* The rule can have output or output list type attributes. In that case the + output files come from the actual attribute values. + [See example](cookbook.md#outputs-custom) + +Each output file must have exactly one generating action. See the +[library](lib/ctx.html#outputs) for more context. + +## Default outputs + +Every rule has a set of default outputs. This is used: + +* When the user runs `bazel build` on your target. Bazel will build the default + outputs of the rule. + +* When the target is used as a dependency of another rule. A rule can access + the default outputs by using [target.files](lib/Target.html#files). + This is the case, for example, if you use a rule in the `srcs` attribute of a + `genrule`. + +Use the `files` provider to specify the default outputs of a rule. +If left unspecified, it will contain all the declared outputs. + +```python +def _impl(ctx): + # ... + return DefaultInfo(files=depset([file1, file2])) +``` + +This can be useful for exposing files generated with +[ctx.new_file](lib/ctx.html#new_file). You can also have "implicit +outputs", i.e., files that are declared in the rule, but not in the default +outputs (like `_deploy.jar` in `java_binary`). + +## Actions + +An action describes how to generate a set of outputs from a set of inputs, for +example "run gcc on hello.c and get hello.o". When an action is created, Bazel +doesn't run the command immediately. It registers it in a graph of dependencies, +because an action can depend on the output of another action (e.g. in C, +the linker must be called after compilation). In the execution phase, Bazel +decides which actions must be run and in which order. + +There are three ways to create actions: + +* [ctx.action](lib/ctx.html#action), to run a command. +* [ctx.file_action](lib/ctx.html#file_action), to write a string to a file. +* [ctx.template_action](lib/ctx.html#template_action), to generate a file from a template. + +Actions take a set (which can be empty) of input files and generate a (non-empty) +set of output files. +The set of input and output files must be known during the +[analysis phase](concepts.md#evaluation-model). It might depend on the value +of attributes and information from dependencies, but it cannot depend on the +result of the execution. For example, if your action runs the unzip command, you +must specify which files you expect to be inflated (before running unzip). + +Actions are comparable to pure functions: They should depend only on the +provided inputs, and avoid accessing computer information, username, clock, +network, or I/O devices (except for reading inputs and writing outputs). This is +important because the output will be cached and reused. + +**If an action generates a file that is not listed in its outputs**: This is +fine, but the file will be ignored and cannot be used by other rules. + +**If an action does not generate a file that is listed in its outputs**: This is +an execution error and the build will fail. This happens for instance when a +compilation fails. + +**If an action generates an unknown number of outputs and you want to keep them +all**, you must group them in a single file (e.g., a zip, tar, or other +archive format). This way, you will be able to deterministically declare your +outputs. + +**If an action does not list a file it uses as an input**, the action execution +will most likely result in an error. The file is not guaranteed to be available +to the action, so if it **is** there, it's due to coincidence or error. + +**If an action lists a file as an input, but does not use it**: This is fine. +However, it can affect action execution order, resulting in sub-optimal +performance. + +Dependencies are resolved by Bazel, which will decide which actions are +executed. It is an error if there is a cycle in the dependency graph. Creating +an action does not guarantee that it will be executed: It depends on whether +its outputs are needed for the build. + +## Configurations + +Imagine that you want to build a C++ binary and target a different architecture. +The build can be complex and involve multiple steps. Some of the intermediate +binaries, like the compilers and code generators, have to run on your machine +(the host); some of the binaries such the final output must be built for the +target architecture. + +For this reason, Bazel has a concept of "configurations" and transitions. The +topmost targets (the ones requested on the command line) are built in the +"target" configuration, while tools that should run locally on the host are +built in the "host" configuration. Rules may generate different actions based on +the configuration, for instance to change the cpu architecture that is passed to +the compiler. In some cases, the same library may be needed for different +configurations. If this happens, it will be analyzed and potentially built +multiple times. + +By default, Bazel builds the dependencies of a target in the same configuration +as the target itself, i.e. without transitioning. When a target depends on a +tool, the label attribute will specify a transition to the host configuration. +This causes the tool and all of its dependencies to be built for the host +machine, assuming those dependencies do not themselves have transitions. + +For each [label attribute](lib/attr.html#label), you can decide whether the +dependency should be built in the same configuration, or transition to the host +configuration (using `cfg`). If a label attribute has the flag +`executable=True`, the configuration must be set explicitly. +[See example](cookbook.html#execute-a-binary) + +In general, sources, dependent libraries, and executables that will be needed at +runtime can use the same configuration. + +Tools that are executed as part of the build (e.g., compilers, code generators) +should be built for the host configuration. In this case, specify `cfg="host"` +in the attribute. + +Otherwise, executables that are used at runtime (e.g. as part of a test) should +be built for the target configuration. In this case, specify `cfg="target"` in +the attribute. + +The configuration `"data"` is present for legacy reasons and should be used for +the `data` attributes. + +## <a name="fragments"></a> Configuration Fragments + +Rules may access [configuration fragments](lib/skylark-configuration-fragment.html) +such as `cpp`, `java` and `jvm`. However, all required fragments must be +declared in order to avoid access errors: + +```python +def _impl(ctx): + # Using ctx.fragments.cpp would lead to an error since it was not declared. + x = ctx.fragments.java + ... + +my_rule = rule( + implementation = _impl, + fragments = ["java"], # Required fragments of the target configuration + host_fragments = ["java"], # Required fragments of the host configuration + ... +) +``` + +`ctx.fragments` only provides configuration fragments for the target +configuration. If you want to access fragments for the host configuration, +use `ctx.host_fragments` instead. + +## Providers + +Providers are pieces of information that a rule exposes to other rules that +depend on it. This data can include output files, libraries, parameters to pass +on a tool's command line, or anything else the depending rule should know about. +Providers are the only mechanism to exchange data between rules, and can be +thought of as part of a rule's public interface (loosely analogous to a +function's return value). + +A rule can only see the providers of its direct dependencies. If there is a rule +`top` that depends on `middle`, and `middle` depends on `bottom`, then we say +that `middle` is a direct dependency of `top`, while `bottom` is a transitive +dependency of `top`. In this case, `top` can see the providers of `middle`. The +only way for `top` to see any information from `bottom` is if `middle` +re-exports this information in its own providers; this is how transitive +information can be accumulated from all dependencies. In such cases, consider +using [depsets](depsets.md) to hold the data more efficiently without excessive +copying. + +Providers can be declared using the [provider()](lib/globals.html#provider) function: + +```python +TransitiveDataInfo = provider() +``` + +Rule implementation function can then construct and return provider instances: + +```python +def rule_implementation(ctx): + ... + return [TransitiveDataInfo(value = ["a", "b", "c"])] +``` + +`TransitiveDataInfo` acts both as a constructor for provider instances and as a key to access them. +A [target](lib/Target.html) serves as a map from each provider that the target supports, to the +target's corresponding instance of that provider. +A rule can access the providers of its dependencies using the square bracket notation (`[]`): + +```python +def dependent_rule_implementation(ctx): + ... + s = depset() + for dep_target in ctx.attr.deps: + s += dep_target[TransitiveDataInfo].value + ... +``` + +All targets have a [`DefaultInfo`](lib/globals.html#DefaultInfo) provider that can be used to access +some information relevant to all targets. + +Providers are only available during the analysis phase. Examples of usage: + +* [mandatory providers](cookbook.md#mandatory-providers) +* [optional providers](cookbook.md#optional-providers) + +> *Note:* +> Historically, Bazel also supported provider instances that are identified by strings and +> accessed as fields on the `target` object instead of as keys. This style is deprecated +> but still supported. Return legacy providers as follows: +> +```python +def rule_implementation(ctx): + ... + modern_provider = TransitiveDataInfo(value = ["a", "b", "c"]) + # Legacy style. + return struct(legacy_provider = struct(...), + another_legacy_provider = struct(...), + # The `providers` field contains provider instances that can be accessed + # the "modern" way. + providers = [modern_provider]) +``` +> To access legacy providers, use the dot notation. +> Note that the same target can define both modern and legacy providers: +> +```python +def dependent_rule_implementation(ctx): + ... + s = depset() + for dep_target in ctx.attr.deps: + x = dep_target.legacy_provider # legacy style + s += dep_target[TransitiveDataInfo].value # modern style + ... +``` +> **We recommend using modern providers for all future code.** + +## Runfiles + +Runfiles are a set of files used by the (often executable) output of a rule +during runtime (as opposed to build time, i.e. when the binary itself is +generated). +During the [execution phase](concepts.md#evaluation-model), Bazel creates a +directory tree containing symlinks pointing to the runfiles. This stages the +environment for the binary so it can access the runfiles during runtime. + +Runfiles can be added manually during rule creation and/or collected +transitively from the rule's dependencies: + +```python +def _rule_implementation(ctx): + ... + transitive_runfiles = depset() + for dep in ctx.attr.special_dependencies: + transitive_runfiles += dep.transitive_runtime_files + + runfiles = ctx.runfiles( + # Add some files manually. + files = [ctx.file.some_data_file], + # Add transitive files from dependencies manually. + transitive_files = transitive_runfiles, + # Collect runfiles from the common locations: transitively from srcs, + # deps and data attributes. + collect_default = True, + ) + # Add a field named "runfiles" to the DefaultInfo provider in order to actually + # create the symlink tree. + return [DefaultInfo(runfiles=runfiles)] +``` + +Note that non-executable rule outputs can also have runfiles. For example, a +library might need some external files during runtime, and every dependent +binary should know about them. + +Also note that if an action uses an executable, the executable's runfiles can +be used when the action executes. + +Normally, the relative path of a file in the runfiles tree is the same as the +relative path of that file in the source tree or generated output tree. If these +need to be different for some reason, you can specify the `root_symlinks` or +`symlinks` arguments. The `root_symlinks` is a dictionary mapping paths to +files, where the paths are relative to the root of the runfiles directory. The +`symlinks` dictionary is the same, but paths are implicitly prefixed with the +name of the workspace. + +```python + ... + runfiles = ctx.runfiles( + root_symlinks = {"some/path/here.foo": ctx.file.some_data_file2} + symlinks = {"some/path/here.bar": ctx.file.some_data_file3} + ) + # Creates something like: + # sometarget.runfiles/ + # some/ + # path/ + # here.foo -> some_data_file2 + # <workspace_name>/ + # some/ + # path/ + # here.bar -> some_data_file3 +``` + +If `symlinks` or `root_symlinks` is used, be careful not to map two different +files to the same path in the runfiles tree. This will cause the build to fail +with an error describing the conflict. To fix, you will need to modify your +`ctx.runfiles` arguments to remove the collision. This checking will be done for +any targets using your rule, as well as targets of any kind that depend on those +targets. + +## Output groups + +By default Bazel builds a target's +[default outputs](#default-outputs). However, a rule can also create + other outputs that are not part of a typical build but might still be useful, + such as debug information files. The facility for this is _output groups_. + +A rule can declare that a certain file belongs to a certain output group by returning +the [OutputGroupInfo](lib/globals.html#OutputGroupInfo) provider. Fields of +that provider are output group names: + +```python +def _impl(ctx): + name = ... + binary = ctx.new_file(name) + debug_file = ctx.new_file(name + ".pdb") + # ... add actions to generate these files + return [DefaultInfo(files = depset([binary])), + OutputGroupInfo(debug_files = depset([debug_file]), + all_files = depset([binary, debug_file]))] +``` + +By default, only the `binary` file will be built. +The user can specify an [`--output_groups=debug_files`](../command-line-reference.html#build) +flag on the command line. In that case, only `debug_file` will be built. If the user +specifies `--output_groups=all_files`, both `binary` and `debug_file` will be build. + +> Note: [OutputGroupInfo](skylark/lib/globals.html#OutputGroupInfo) is a regular +> [provider](#providers), and dependencies of a target can examine it using +> the `target[OutputGroupInfo]` syntax. + +## Code coverage instrumentation + +A rule can use the `instrumented_files` provider to provide information about +which files should be measured when code coverage data collection is enabled: + +```python +def _rule_implementation(ctx): + ... + return struct(instrumented_files = struct( + # Optional: File extensions used to filter files from source_attributes. + # If not provided, then all files from source_attributes will be + # added to instrumented files, if an empty list is provided, then + # no files from source attributes will be added. + extensions = ["ext1", "ext2"], + # Optional: Attributes that contain source files for this rule. + source_attributes = ["srcs"], + # Optional: Attributes for dependencies that could include instrumented + # files. + dependency_attributes = ["data", "deps"])) +``` + +[ctx.config.coverage_enabled](lib/configuration.html#coverage_enabled) notes +whether coverage data collection is enabled for the current run in general +(but says nothing about which files specifically should be instrumented). +If a rule implementation needs to add coverage instrumentation at +compile-time, it can determine if its sources should be instrumented with +[ctx.coverage_instrumented](lib/ctx.html#coverage_instrumented): + +```python +# Are this rule's sources instrumented? +if ctx.coverage_instrumented(): + # Do something to turn on coverage for this compile action +``` + +Note that function will always return false if `ctx.config.coverage_enabled` is +false, so you don't need to check both. + +If the rule directly includes sources from its dependencies before compilation +(e.g. header files), it may also need to turn on compile-time instrumentation +if the dependencies' sources should be instrumented. In this case, it may +also be worth checking `ctx.config.coverage_enabled` so you can avoid looping +over dependencies unnecessarily: + +```python +# Are this rule's sources or any of the sources for its direct dependencies +# in deps instrumented? +if ctx.config.coverage_enabled: + if (ctx.coverage_instrumented() or + any(ctx.coverage_instrumented(dep) for dep in ctx.attr.deps): + # Do something to turn on coverage for this compile action +``` + +## Executable rules + +An executable rule is a rule that users can run using `bazel run`. + +To make a rule executable, set `executable=True` in the +[rule function](lib/globals.html#rule). The `implementation` function of the +rule must generate the output file `ctx.outputs.executable`. +[See example](cookbook.md#outputs-executable) + +## Test rules + +Test rules are run using `bazel test`. + +To create a test rule, set `test=True` in the +[rule function](lib/globals.html#rule). The name of the rule must +also end with `_test`. Test rules are implicitly executable, which means that +the `implementation` function of the rule must generate the output file +`ctx.outputs.executable`. + +Test rules inherit the following attributes: `args`, `flaky`, `local`, +`shard_count`, `size`, `timeout`. + +[1]: https://www.python.org/dev/peps/pep-0008/#id46 diff --git a/site/docs/support.md b/site/docs/support.md index 0d1b8cdaa6..407cec8e0c 100644 --- a/site/docs/support.md +++ b/site/docs/support.md @@ -1,4 +1,257 @@ --- -layout: redirect -redirect: docs/support.html +layout: documentation +title: Get Support --- + +# Get Support + +* Ask questions on [Stack Overflow](http://stackoverflow.com/questions/tagged/bazel) using the +`bazel` tag. +* Discuss on the [User mailing list](https://groups.google.com/forum/#!forum/bazel-discuss). +* Report bugs or feature requests in our [GitHub issue tracker](https://github.com/bazelbuild/bazel/issues). +* Find other Bazel contributors on [IRC](http://webchat.freenode.net) +(irc.freenode.net#bazel). + +# Support Policy + +We generally avoid making backwards-incompatible changes. We have several years of experience with +supporting a huge code base that is concurrently worked on by thousands of engineers every day, +and have successfully made significant changes to the core as well as to the rules without missing +a beat. We run hundreds of thousands of tests at Google before every single release to ensure that +it stays that way. + +That said, we occasionally have to make incompatible changes in order to fix bugs, to make further +improvements to the system, such as improving performance or usability, or to lock down APIs that +are known to be brittle. + +This document gives an overview of features that are widely used and that we consider stable. By +stable, we mean that the changes we make will be backwards compatible, or that we will provide a +migration path. + +It also covers features that are unstable. Either they are not yet widely used, or we are already +planning to change them significantly, possibly in ways that are not backwards compatible. + +We cannot cover everything that might change, but you can reasonably expect that we provide +advance notice on the mailing list before a major change happens. We're also happy to provide more +details, just ask on [bazel-discuss](https://groups.google.com/forum/#!forum/bazel-discuss). + +All undocumented features (attributes, rules, "Make" variables, and flags) are subject to change +at any time without prior notice. Features that are documented but marked *experimental* are also +subject to change at any time without prior notice. + +Bazel's extension language Skylark (anything you write in a `.bzl` file) is still subject to change. +We are in the process of migrating Google to Skylark, and expect the language part to extend macros +to stabilize as part of that process. Adding rules with skylark is still somewhat experimental. + +Help keep us honest: report bugs and regressions in the +[GitHub bugtracker](https://github.com/bazelbuild/bazel/issues). We will make an effort to triage all +reported issues within 2 business days. + +## Releases + +We regularly publish [binary releases of Bazel](https://github.com/bazelbuild/bazel/releases). To +that end, we announce release candidates on +[bazel-discuss](https://groups.google.com/forum/#!forum/bazel-discuss); these are binaries that have +passed all of our unit tests. Over the next few days, we regression test all applicable build +targets at Google. If you have a critical project using Bazel, we recommend that you establish an +automated testing process that tracks the current release candidate, and report any regressions. + +If no regressions are discovered, we officially release the candidate after a week. However, +regressions can delay the release of a release candidate. If regressions are found, we apply +corresponding cherry-picks to the release candidate to fix those regressions. If no further +regressions are found for two business days, but not before a week has elapsed since the first +release candidate, we release it. + +### Release versioning + +Version 0.1 is our first release marking the start of our beta phase. Until version 1.0.0, we +increase the MINOR version every time we reach a [new milestone](http://bazel.build/roadmap.html). + +Version 1.0.0 marks the end of our beta phase; afterwards, we will label each release with a +version number of the form MAJOR.MINOR.PATCH according to the +[semantic version 2.0.0 document](http://semver.org). + +## Current Status + +### Built-In Rules and the Internal API For Rules ### +We are planning a number of changes to the APIs between the core of Bazel and the built-in rules, +in order to make it easier for us to develop openly. This has the added benefit of also making it +easier for users to maintain their own rules (if written in Java instead of Skylark), if they don't +want to or cannot check this code into our repository. However, it also means our internal API is +not stable yet. In the long term, we want to move to Skylark wholesale, so we encourage contributors +to use Skylark instead of Java when writing new rules. Rewriting all of our built-in rules is going +to be a lengthy process however. + +1. We will fix the friction points that we know about, as well as those that we discover every time + we make changes that span both the internal and external depots. +2. We will drive a number of pending API cleanups to completion, as well as run anticipated cleanups + to the APIs, such as disallowing access to the file system from rule implementations (because + it's not hermetic). +3. We will enumerate the internal rule APIs, and make sure that they are appropriately marked (for + example with annotations) and documented. Just collecting a list will likely give us good + suggestions for further improvements, as well as opportunities for a more principled API review + process. +4. We will automatically check rule implementations against an API whitelist, with the intention + that API changes are implicitly flagged during code review. +5. We will work on removing (legacy) Google-internal features to reduce the amount of differences + between the internal and external rule sets. +6. We will encourage our engineers to make changes in the external depot first, and migrate them to + to the internal one afterwards. +7. We will move more of our rule implementations into the open source repository (Android, Go, + Python, the remaining C++ rules), even if we don't consider the code to be *ready* or if they are + still missing tools to work properly. +8. In order to be able to accept external contributions, our highest priority item for Skylark is a + testing framework. We encourage to write new rules in Skylark rather than in Java. + + +### Stable +We expect the following rules and features to be stable. They are widely used within Google, so +our internal testing should ensure that there are no major breakages. + +<table class="table table-condensed table-striped table-bordered"> + <colgroup> + <col class="support-col-rules" /> + <col class="support-col-notes" /> + </colgroup> + <thead> + <tr> + <th>Rules</th> + <th>Notes</th> + </tr> + </thead> + <tbody> + <tr> + <td>C/C++ rules except <code>cc_toolchain</code> and <code>cc_toolchain_suite</code></td> + <td></td> + </tr> + <tr> + <td>Java rules except <code>java_toolchain</code></td> + <td></td> + </tr> + <tr> + <td>Android rules except <code>android_ndk_repository</code> and + <code>android_sdk_repository</code></td> + <td></td> + </tr> + <tr> + <td><code>genrule</code></td> + <td></td> + </tr> + <tr> + <td><code>genquery</code></td> + <td></td> + </tr> + <tr> + <td><code>test_suite</code></td> + <td></td> + </tr> + <tr> + <td><code>filegroup</code></td> + <td></td> + </tr> + <tr> + <td><code>config_setting</code></td> + <td> + <ul> + <li>This rule is used in <code>select()</code> expressions. We have hundreds of uses, so + we expect the basic functionality to be stable. That said, there are some common use + cases that are not covered yet, or that require workarounds. For example, it's not + easily possible to select on information specified in a CROSSTOOL file, such as the + target abi. Another example is that it's not possible to OR multiple conditions, + leading to duplicated entries in the select. + </li> + </ul> + </td> + </tr> + </tbody> +</table> + + +### Unstable +These rules and features have known limitations that we will likely address in future releases. + +<table class="table table-condensed table-striped table-bordered"> + <colgroup> + <col class="support-col-rules" /> + <col class="support-col-notes" /> + </colgroup> + <thead> + <tr> + <th>Feature</th> + <th>Notes</th> + </tr> + </thead> + <tbody> + <tr> + <td><code>cc_toolchain</code> and <code>cc_toolchain_suite</code></td> + <td> + <ul> + <li>We intend to make significant changes to the way C/C++ toolchains are defined; we will + keep our published C/C++ toolchain definition(s) up to date, but we make no guarantees for + custom ones.</li> + </ul> + </td> + </tr> + <tr> + <td>iOS/Objective C rules</td> + <td> + <ul> + <li>We cannot vouch for changes made by Apple ® to the underlying tools and + infrastructure.</li> + <li>The rules are fairly new and still subject to change; we try to avoid breaking changes, + but this may not always be possible.</li> + <li>No testing support yet.</li> + </ul> + </td> + </tr> + <tr> + <td>Python rules</td> + <td> + <ul> + <li>The rules support neither Python 3, C/C++ extensions, nor packaging. + </li> + </ul> + </td> + </tr> + <tr> + <td>Extra actions (<code>extra_action</code>, <code>action_listener</code>)</td> + <td> + <ul> + <li>Extra actions expose information about Bazel that we consider to be implementation + details, such as the exact interface between Bazel and the tools we provide; as such, + users will need to keep up with changes to tools to avoid breakage.</li> + </ul> + </td> + </tr> + <tr> + <td><code>environment_group</code></td> + <td> + <ul> + <li>We're planning to use it more extensively, replacing several machine-enforceable + constraint mechanism, but there's only a handful of uses so far. We fully expect it to + work, but there's a small chance that we have to go back to the drawing board.</li> + </ul> + </td> + </tr> + <tr> + <td><code>android_ndk_repository</code> and <code>android_sdk_repository</code></td> + <td> + <ul> + <li>We don't support pre-release NDKs or SDKs at this time. Furthermore, we may still + make backwards-incompatible changes to the attributes or the semantics.</li> + </ul> + </td> + </tr> + <tr> + <td><code>Fileset</code></td> + <td> + <ul> + <li>There are vestiges of Fileset / FilesetEntry in the source code, but we do not intend to + support them in Bazel, ever.</li> + <li>They're still widely used internally, and are therefore unlikely to go away in the near + future.</li> + </ul> + </td> + </tbody> +</table> + diff --git a/site/versions/master/docs/test-encyclopedia.html b/site/docs/test-encyclopedia.html index 76e326283b..76e326283b 100644 --- a/site/versions/master/docs/test-encyclopedia.html +++ b/site/docs/test-encyclopedia.html diff --git a/site/docs/test-encyclopedia.md b/site/docs/test-encyclopedia.md deleted file mode 100644 index b87cc71ac7..0000000000 --- a/site/docs/test-encyclopedia.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/test-encyclopedia.html ---- diff --git a/site/docs/tutorial/android-app.md b/site/docs/tutorial/android-app.md index ea52e3ef1d..f31736b9f9 100644 --- a/site/docs/tutorial/android-app.md +++ b/site/docs/tutorial/android-app.md @@ -1,4 +1,295 @@ --- -layout: redirect -redirect: docs/tutorial/android-app.html +layout: documentation +title: Tutorial - Build an Android App --- + +# Tutorial - Build an Android App + +The sample Android app in this tutorial is a very simple application that makes +an HTTP connection to the [backend server](backend-server.md) and displays the +resulting response. + +Here, you'll do the following: + +* Review the source files for the app +* Update the `WORKSPACE` file +* Create a `BUILD` file +* Run the build +* Find the build outputs +* Run the app + +## Review the source files + +Let's take a look at the source files for the app. These are located in +`$WORKSPACE/android/`. + +The key files and directories are: + +<table class="table table-condensed table-striped"> +<thead> +<tr> +<td>Name</td> +<td>Location</td> +</tr> +</thead> +<tbody> +<tr> +<td>Manifest file</td> +<td><code>src/main/java/com/google/bazel/example/android/AndroidManifest.xml</code></td> +</tr> +<tr> +<td>Activity source file</td> +<td><code>src/main/java/com/google/bazel/example/android/activities/MainActivity.java</code></td> +</tr> +<tr> +<td>Resource file directory</td> +<td><code>src/main/java/com/google/bazel/example/android/res/</code></td> +</tr> +</tbody> +</table> + +Note that you're just looking at these files now to become familiar with the +structure of the app. You don't have to edit any of the source files to complete +this tutorial. + +## Update the WORKSPACE file + +Bazel needs to run the Android SDK +[build tools](https://developer.android.com/tools/revisions/build-tools.html) +and uses the SDK libraries to build the app. This means that you need to add +some information to your `WORKSPACE` file so that Bazel knows where to find +them. Note that this step is not required when you build for other platforms. +For example, Bazel automatically detects the location of Java, C++ and +Objective-C compilers from settings in your environment. + +Add the following lines to your `WORKSPACE` file: + +```python +android_sdk_repository( + name = "androidsdk", + # Replace with your installed Android SDK API level + api_level = 25 +) +``` + +This will use the Android SDK specified referenced by the `ANDROID_HOME` +environment variable, and automatically detect the latest build tools +version installed within that location. + +Alternatively, you can explicitly specify the location of the Android +SDK and build tools version to use by including the `path` and +`build_tools_version` attributes: + +```python +android_sdk_repository( + name = "androidsdk", + path = "/path/to/Android/sdk", + api_level = 25, + build_tools_version = "25.0.1" +) +``` + +**Optional:** This is not required by this tutorial, but if you want to compile +native code into your Android app, you also need to download the +[Android NDK](https://developer.android.com/ndk/downloads/index.html) and +tell Bazel where to find it by adding the following rule to your `WORKSPACE` +file: + +```python +android_ndk_repository( + name = "androidndk", + # Replace with the Android NDK API level + api_level = 21 +) +``` + +`api_level` is the version of the Android API the SDK and the NDK target +(for example, 19 for Android K and 21 for Android L). It's not necessary to set +the API levels to the same value for the SDK and NDK. +[This web page](https://developer.android.com/ndk/guides/stable_apis.html) +contains a map from Android releases to NDK-supported API levels. + +Similar to `android_sdk_repository`, the path to the Android NDK is inferred from +the `ANDROID_NDK_HOME` environment variable by default. The path can also be +explicitly specified with a `path` attribute on `android_ndk_repository`. + +## Create a BUILD file + +A [`BUILD` file](/docs/build-ref.html#BUILD_files) is a text file that describes +the relationship between a set of build outputs -- for example, compiled +software libraries or executables -- and their dependencies. These dependencies +may be source files in your workspace or other build outputs. `BUILD` files are +written in the Bazel *build language*. + +`BUILD` files are part of concept in Bazel known as the *package hierarchy*. +The package hierarchy is a logical structure that overlays the directory +structure in your workspace. Each [package](/docs/build-ref.html#packages) is a +directory (and its subdirectories) that contains a related set of source files +and a `BUILD` file. The package also includes any subdirectories, excluding +those that contain their own `BUILD` file. The *package name* is the name of the +directory where the `BUILD` file is located. + +Note that this package hierarchy is distinct from, but coexists with, the Java +package hierarchy for your Android app. + +For the simple Android app in this tutorial, we'll consider all the source files +in `$WORKSPACE/android/` to comprise a single Bazel package. A more complex +project may have many nested packages. + +At a command-line prompt, open your new `BUILD` file for editing: + +```bash +vi $WORKSPACE/android/BUILD +``` + +### Add an android_library rule + +A `BUILD` file contains several different types of instructions for Bazel. The +most important type is the [build rule](/docs/build-ref.html#funcs), which tells +Bazel how to build an intermediate or final software output from a set of source +files or other dependencies. + +Bazel provides two build rules, `android_library` and `android_binary`, that you +can use to build an Android app. For this tutorial, you'll first use the +[`android_library`](/docs/be/android.html#android_library) rule to tell +Bazel how to build an +[Android library module](http://developer.android.com/tools/projects/index.html#LibraryProjects) +from the app source code and resource files. Then you'll use the +`android_binary` rule to tell it how to build the Android application package. + +Add the following to your `BUILD` file: + +```python +android_library( + name = "activities", + srcs = glob(["src/main/java/com/google/bazel/example/android/activities/*.java"]), + custom_package = "com.google.bazel.example.android.activities", + manifest = "src/main/java/com/google/bazel/example/android/activities/AndroidManifest.xml", + resource_files = glob(["src/main/java/com/google/bazel/example/android/activities/res/**"]), +) +``` + +As you can see, the `android_library` build rule contains a set of attributes +that specify the information that Bazel needs to build a library module from the +source files. Note also that the name of the rule is `activities`. You'll +reference the rule using this name as a dependency in the `android_binary` rule. + +### Add an android_binary rule + +The [`android_binary`](/docs/be/android.html#android_binary) rule builds +the Android application package (`.apk` file) for your app. + +Add the following to your build file: + +```python +android_binary( + name = "android", + custom_package = "com.google.bazel.example.android", + manifest = "src/main/java/com/google/bazel/example/android/AndroidManifest.xml", + resource_files = glob(["src/main/java/com/google/bazel/example/android/res/**"]), + deps = [":activities"], +) +``` + +Here, the `deps` attribute references the output of the `activities` rule you +added to the `BUILD` file above. This means that, when Bazel builds the output +of this rule, it checks first to see if the output of the `activities` library +rule has been built and is up-to-date. If not, it builds it and then uses that +output to build the application package file. + +Now, save and close the file. You can compare your `BUILD` file to the +[completed example](https://github.com/bazelbuild/examples/blob/master/tutorial/android/BUILD) +in the `master` branch of the GitHub repo. + +## Run the build + +You use the +[`bazel`](/docs/bazel-user-manual.html) command-line tool to run builds, execute +unit tests and perform other operations in Bazel. This tool is located in the +`output` subdirectory of the location where you installed Bazel. During +[installation](/docs/install.md), you probably added this location to your +path. + +Before you build the sample app, make sure that your current working directory +is inside your Bazel workspace: + +```bash +cd $WORKSPACE +``` + +Now, enter the following to build the sample app: + +```bash +bazel build //android:android +``` + +The [`build`](/docs/bazel-user-manual.html#build) subcommand instructs Bazel to +build the target that follows. The target is specified as the name of a build +rule inside a `BUILD` file, with along with the package path relative to +your workspace directory. Note that you can sometimes omit the package path +or target name, depending on your current working directory at the command +line and the name of the target. See [Labels](/docs/build-ref.html#labels) in +*Bazel Concepts and Terminology* page for more information about target labels +and paths. + +Bazel now launches and builds the sample app. During the build process, its +output will appear similar to the following: + +```bash +INFO: Found 1 target... +Target //android:android up-to-date: + bazel-bin/android/android_deploy.jar + bazel-bin/android/android_unsigned.apk + bazel-bin/android/android.apk +INFO: Elapsed time: 7.237s, Critical Path: 5.81s +``` + +## Find the build outputs + +Bazel stores the outputs of both intermediate and final build operations in +a set of per-user, per-workspace output directories. These directories are +symlinked from the following locations: + +* `$WORKSPACE/bazel-bin`, which stores binary executables and other runnable + build outputs +* `$WORKSPACE/bazel-genfiles`, which stores intermediary source files that are + generated by Bazel rules +* `$WORKSPACE/bazel-out`, which stores other types of build outputs + +Bazel stores the Android `.apk` file generated using the `android_binary` rule +in the `bazel-bin/android/` directory, where the subdirectory name `android` is +derived from the name of the Bazel package. + +At a command prompt, list the contents of this directory and find the +`android.apk` file: + +```bash +ls $WORKSPACE/bazel-bin/android +``` + +## Run the app + +You can now deploy the app to a connected Android device or emulator from the +command line using the +[`bazel mobile-install`](http://bazel.build/docs/bazel-user-manual.html#mobile-install) +command. This command uses the Android Debug Bridge (`adb`) to communicate with +the device. You must set up your device to use `adb` following the instructions +in +[Android Debug Bridge](http://developer.android.com/tools/help/adb.html) before +deployment. + +Enter the following: + +```bash +bazel mobile-install //android:android +``` + +Note that the `mobile-install` subcommand also supports the +[`--incremental`](http://bazel.build/docs/bazel-user-manual.html#mobile-install) +flag that can be used to deploy only those parts of the app that have changed +since the last deployment. + +## What's next + +Now that you've built a sample app for Android, it's time to do the same for +the [iOS app](ios-app.md). diff --git a/site/docs/tutorial/app.md b/site/docs/tutorial/app.md index d3c2872f6f..b3b274c2d5 100644 --- a/site/docs/tutorial/app.md +++ b/site/docs/tutorial/app.md @@ -1,4 +1,45 @@ --- -layout: redirect -redirect: docs/tutorial/app.html +layout: documentation +title: Build Mobile Application --- + +# Build Mobile Application + +You can use Bazel to build a variety of software outputs, including +Linux and macOS (OS X) applications written in Java, C++ and Objective-C. You can +also use Bazel to build software for other platforms or written in other +languages. + +This tutorial shows how to use Bazel to build the following: + +* An Android app +* An iOS app +* A mobile backend server running on App Engine + +In this tutorial, you'll learn how to: + +* Set up a Bazel workspace and create a `WORKSPACE` file +* Create `BUILD` files that contain the instructions used by Bazel to build + the software +* Run builds using the Bazel command line tool + +## Requirements + +You can follow the steps in this tutorial on either a Linux or Mac OS X system. +However, you can only build the iOS app if you are running Bazel on OS X. If +you are using Linux, you can skip the iOS instructions and still complete +the rest of the tutorial steps. + +## Sample project + +You don't have to write your own mobile apps and backend server to use this +tutorial. Instead, you'll use a sample project hosted on GitHub. The sample +project is hosted at the following location: + +[https://github.com/bazelbuild/examples/](https://github.com/bazelbuild/examples/) + +You'll grab the sample project files in the next step in this tutorial. + +## What's next + +Let's start off by [setting up](environment.md) the tutorial environment. diff --git a/site/docs/tutorial/backend-server.md b/site/docs/tutorial/backend-server.md index 150f9152ec..00c61ac43d 100644 --- a/site/docs/tutorial/backend-server.md +++ b/site/docs/tutorial/backend-server.md @@ -1,4 +1,237 @@ --- -layout: redirect -redirect: docs/tutorial/backend-server.html +layout: documentation +title: Tutorial - Build the Backend Server --- + +# Tutorial - Build the Backend Server + +The backend server is a simple web application that runs on Google App Engine +and responds to incoming HTTP requests from the sample Android and iOS apps. + +Here, you'll do the following: + +* Review the source files for the app +* Update the `WORKSPACE` file +* Create a `BUILD` file +* Run the build +* Find the build outputs +* Run the application on a local development server +* Deploy to Google App Engine + +Bazel provides a set of [App Engine build rules](https://github.com/bazelbuild/rules_appengine) +written using the [Skylark](/docs/skylark/index.html) framework. You'll use +these in the steps below to build the application. + +## Review the source files + +The source files for the backend server are located in `$WORKSPACE/backend/`. + +The key files and directories are: + +<table class="table table-condensed table-striped"> +<thead> +<tr> +<td>Name</td> +<td>Location</td> +</tr> +</thead> +<tbody> +<tr> +<td>Source file directory</td> +<td><code>src/main/java/com/google/bazel/example/app/</code></td> +</tr> +<tr> +<td>Web application metadata directory</td> +<td><code>webapp/WEB-INF/</code></td> +</tr> +</tbody> +</table> + +## Update the WORKSPACE file + +As with the Android app, you must add references to +[external dependencies](http://bazel.build/docs/external.html) to your `WORKSPACE` +file. For the backend server, these are references to the App Engine SDK, +the Java Servlet SDK and other libraries needed to build the App Engine +applications. + +### Add the App Engine rule + +When you built the Android app, you added a reference to the location on your +filesystem where you downloaded and installed the Android SDK. For the +backend server, however, you'll give Bazel instructions for downloading the +required App Engine SDK package from a remote server. This is optional. You +can also download and install the SDK manually on your filesystem and reference +it from that location as described in the +[App Engine rule documentation](https://github.com/bazelbuild/rules_appengine). + +Add the following to your `WORKSPACE` file: + +```python +http_archive( + name = "io_bazel_rules_appengine", + sha256 = "f4fb98f31248fca5822a9aec37dc362105e57bc28e17c5611a8b99f1d94b37a4", + strip_prefix = "rules_appengine-0.0.6", + url = "https://github.com/bazelbuild/rules_appengine/archive/0.0.6.tar.gz", +) +load("@io_bazel_rules_appengine//appengine:appengine.bzl", "appengine_repositories") +appengine_repositories() +``` + +[`http_archive`](/docs/be/workspace.html#http_archive) downloads the +AppEngine rules from a GitHub archive. We could also have used +[`git_repository`](/docs/be/workspace.html#git_repository) to fetch the rules +directly from the Git repository. +Then the next two lines use the `appengine_repositories` function defined in +these rules to download the libraries and SDK needed to build AppEngine +applications. + +Now, save and close the file. You can compare your `WORKSPACE` file to the +[completed example](https://github.com/bazelbuild/examples//blob/master/tutorial/WORKSPACE) +in the `master` branch of the GitHub repo. + +## Create a BUILD file + +Now that you have set up the external dependencies, you can go ahead and create +the `BUILD` file for the backend server, as you did previously for the sample +Android and iOS apps. + +Open your new `BUILD` file for editing: + +```bash +vi $WORKSPACE/backend/BUILD +``` + +### Add a java_binary rule + +Add the following to your `BUILD` file: + +```python +java_binary( + name = "app", + srcs = glob(["src/main/java/**/*.java"]), + main_class = "does.not.exist", + deps = [ + "@io_bazel_rules_appengine//appengine:javax.servlet.api", + ], +) +``` + +The [`java_binary`](/docs/be/java.html#java_binary) tells Bazel +how to build a Java `.jar` library for your application, plus a wrapper shell +script that launches the application code from the specified main class. Here, +we're using this rule instead of the +[`java_library`](/docs/be/java.html#java_library) because we need +the `.jar` file to contain all the dependencies required to build the final +App Engine `.war` file. For this reason, we specify a bogus class name +for the `main_class` attribute. + +### Add an appengine_war rule + +Add the following to your `BUILD` file: + +```python +load("@io_bazel_rules_appengine//appengine:appengine.bzl", "appengine_war") + +appengine_war( + name = "backend", + data = [":webapp"], + data_path = "/backend/webapp", + jars = [":app_deploy.jar"], +) + +filegroup( + name = "webapp", + srcs = glob(["webapp/**/*"]), +) +``` + +The [`appengine_war`](/docs/be/appengine.html#appengine_war) +rule builds the final App Engine `war` file from the library `.jar` file and web +application metadata files in the `webapp` directory. + +Save and close the file. Again, the +[completed example](https://github.com/google/bazel-examples/blob/master/tutorial/backend/BUILD) +is in the `master` branch of the GitHub repo. + +## Run the build + +Make sure that your current working directory is inside your Bazel workspace: + +```bash +cd $WORKSPACE +``` + +Now, enter the following to build the sample app: + +```bash +bazel build //backend:backend +``` + +Bazel now launches and builds the sample app. During the build process, its +output will appear similar to the following: + +```bash +INFO: Found 1 target... +Target //backend:backend up-to-date: + bazel-bin/backend/backend.war + bazel-bin/backend/backend.deploy + bazel-bin/backend/backend +INFO: Elapsed time: 56.867s, Critical Path: 2.72s +``` + +## Find the build outputs + +The `.war` file and other outputs are located in the +`$WORKSPACE/bazel-bin/backend` directory. + +In particular, the `appengine_war` rule generates scripts that you can use to +run your backend locally or deploy it to Google App Engine: + +## Run the application on a local development server + +Here, you'll start a local App Engine development server in your environment and +run your application on it. + +To run the application, enter the following: + +```bash +bazel-bin/backend/backend --port=12345 +``` + +Your application will be available at `http://localhost:12345` + +## Deploy to Google App Engine + +You can also deploy the application to the live App Engine serving +environment on Google Cloud Platform. For this scenario, you must first create +a new Cloud Platform project and App Engine application using the Google Cloud +Platform Console. +Follow [this link](https://console.cloud.google.com/projectselector/appengine/create?lang=java&st=true) +to perform these actions. + +Build the target that allows to deploy to App Engine: + +```bash +bazel build --java_toolchain=@io_bazel_rules_appengine//appengine:jdk7 //backend:backend.deploy +``` + +Then, to deploy the application, enter the following: + +```bash +bazel-bin/backend/backend.deploy <project-id> +``` + +The deployment script prompts you to authorize access to Google Cloud Platform. +After you have authorized access the first time, you can deploy the application +using the `bazel` command and the following rule target: + +```bash +bazel run //backend:backend.deploy <project-id> +``` + +Your application URL will be `http://<project-id>.appspot.com`. + +## What's next + +Now let's [review](review.md) the tutorial steps. diff --git a/site/docs/tutorial/cpp.md b/site/docs/tutorial/cpp.md index 6ec737f93d..23362010ef 100644 --- a/site/docs/tutorial/cpp.md +++ b/site/docs/tutorial/cpp.md @@ -1,4 +1,395 @@ --- -layout: redirect -redirect: docs/tutorial/cpp.html +layout: documentation +title: Build C++ --- + +Build C++ +========= + +You can use Bazel to build your C++ application. In this tutorial you'll learn how to: + +* Build your first C++ target +* Use external libraries +* Write and run C++ tests +* Use precompiled libraries + +## Setting up your workspace + +Suppose that you have an existing project in a directory, say, +`~/gitroot/my-project/`. Create an empty file at +`~/gitroot/my-project/WORKSPACE` to show Bazel where your project's root is. +We are going to create a small hello world project with the following directory structure: +{% highlight bash %} +└── my-project + ├── lib + │ ├── BUILD + │ ├── hello-greet.cc + │ └── hello-greet.h + ├── main + │ ├── BUILD + │ ├── hello-time.cc + │ ├── hello-time.h + │ └── hello-world.cc + └── WORKSPACE +{% endhighlight %} + +## Creating source files + +Using the following commands to create the necessary source files: +{% highlight bash %} +# If you're not already there, move to your workspace directory. +cd ~/gitroot/my-project +mkdir ./main +cat > main/hello-world.cc <<'EOF' + +#include "lib/hello-greet.h" +#include "main/hello-time.h" +#include <iostream> +#include <string> + +int main(int argc, char** argv) { + std::string who = "world"; + if (argc > 1) { + who = argv[1]; + } + std::cout << get_greet(who) <<std::endl; + print_localtime(); + return 0; +} +EOF + +cat > main/hello-time.h <<'EOF' + +#ifndef MAIN_HELLO_TIME_H_ +#define MAIN_HELLO_TIME_H_ + +void print_localtime(); + +#endif +EOF + +cat > main/hello-time.cc <<'EOF' + +#include "main/hello-time.h" +#include <ctime> +#include <iostream> + +void print_localtime() { + std::time_t result = std::time(nullptr); + std::cout << std::asctime(std::localtime(&result)); +} +EOF + +mkdir ./lib +cat > lib/hello-greet.h <<'EOF' + +#ifndef LIB_HELLO_GREET_H_ +#define LIB_HELLO_GREET_H_ + +#include <string> + +std::string get_greet(const std::string &thing); + +#endif +EOF + +cat > lib/hello-greet.cc <<'EOF' + +#include "lib/hello-greet.h" +#include <string> + +std::string get_greet(const std::string& who) { + return "Hello " + who; +} +EOF +{% endhighlight %} + +## Adding BUILD files + +As you can see from the source code, `main/hello-world.cc` needs to include both `lib/hello-greet.h` and `main/hello-time.h`. +First we create `lib/BUILD` for hello-greet.cc: + +{% highlight python %} +cc_library( + name = "hello-greet", + srcs = ["hello-greet.cc"], + hdrs = ["hello-greet.h"], + visibility = ["//main:__pkg__"], +) +{% endhighlight %} + +Note that `visibility = ["//main:__pkg__"]` indicates `hello-greet` is visible from `main/BUILD`. +Then we'd create the following `main/BUILD` file: + +{% highlight python %} +cc_library( + name = "hello-time", + srcs = ["hello-time.cc"], + hdrs = ["hello-time.h"], +) + +cc_binary( + name = "hello-world", + srcs = ["hello-world.cc"], + deps = [ + ":hello-time", + "//lib:hello-greet", + ], +) +{% endhighlight %} + +Note when depending on a target in the same package, we can just use `:hello-time`. +When the target is in other package, a full path from root should be used, like `//lib:hello-greet`. + +Now you are ready to build your hello world C++ binary: + +{% highlight bash %} +bazel build main:hello-world +{% endhighlight %} + +This produces the following output: + +{% highlight bash %} +INFO: Found 1 target... +Target //main:hello-world up-to-date: + bazel-bin/main/hello-world +INFO: Elapsed time: 2.869s, Critical Path: 1.00s +{% endhighlight %} + +{% highlight bash %} +./bazel-bin/main/hello-world +{% endhighlight %} + +This produces the following output: + +{% highlight bash %} +Hello world +Thu Jun 23 18:51:46 2016 +{% endhighlight %} + +{% highlight bash %} +./bazel-bin/main/hello-world Bazel +{% endhighlight %} + +This produces the following output: + +{% highlight bash %} +Hello Bazel +Thu Jun 23 18:52:10 2016 +{% endhighlight %} + +Congratulations, you've just built your first Bazel target! + +## Transitive includes + +If a file includes a header, then the file's rule should depend on that header's +library. Conversely, only direct dependencies need to be specified as +dependencies. For example, suppose `sandwich.h` includes `bread.h` and +`bread.h` includes `flour.h`. `sandwich.h` doesn't include `flour.h` (who wants +flour in their sandwich?), so the BUILD file would look like: + +```python +cc_library( + name = "sandwich", + srcs = ["sandwich.cc"], + hdrs = ["sandwich.h"], + deps = [":bread"], +) + +cc_library( + name = "bread", + srcs = ["bread.cc"], + hdrs = ["bread.h"], + deps = [":flour"], +) + +cc_library( + name = "flour", + srcs = ["flour.cc"], + hdrs = ["flour.h"], +) +``` + +Here, the `sandwich` library depends on the `bread` library, which depends +on the `flour` library. + +## Adding include paths + +Sometimes you cannot (or do not want to) base include paths at the workspace +root. Existing libraries might already have a include directory that doesn't +match its path in your workspace. For example, suppose you have the following +directory structure: + +``` +└── my-project + ├── third_party + │ └── some_lib + │ ├── BUILD + │ ├── include + │ │ └── some_lib.h + │ └── some_lib.cc + └── WORKSPACE +``` + +Bazel will expect `some_lib.h` to be included as +`third_party/some_lib/include/some_lib.h`, but suppose `some_lib.cc` includes +`"include/some_lib.h"`. To make that include path valid, +`third_party/some_lib/BUILD` will need to specify that the `some_lib/` +directory is an include directory: + +```python +cc_library( + name = "some_lib", + srcs = ["some_lib.cc"], + hdrs = ["some_lib.h"], + copts = ["-Ithird_party/some_lib"], +) +``` + +This is especially useful for external dependencies, as their header files +must otherwise be included with an `external/[repository-name]/` prefix. + +## Including external libraries + +Suppose you are using [Google Test](https://github.com/google/googletest). You +can use one of the `new_` repository functions in the `WORKSPACE` file to +download Google Test and make it available in your repository: + +```python +new_http_archive( + name = "gtest", + url = "https://github.com/google/googletest/archive/release-1.7.0.zip", + sha256 = "b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0", + build_file = "gtest.BUILD", +) +``` + +Then create `gtest.BUILD`, a BUILD file to use to compile Google Test. +Google Test has several "special" requirements that make its `cc_library` rule +more complicated: + +* `googletest-release-1.7.0/src/gtest-all.cc` `#include`s all of the other files in + `googletest-release-1.7.0/src/`, so we need to exclude it from the compile or we'll get + link errors for duplicate symbols. +* It uses header files that are relative to the `googletest-release-1.7.0/include/` directory + (`"gtest/gtest.h"`), so we must add that directory to the include paths. +* It needs to link in pthread, so we add that as a `linkopt`. + +The final rule looks like this: + +```python +cc_library( + name = "main", + srcs = glob( + ["googletest-release-1.7.0/src/*.cc"], + exclude = ["googletest-release-1.7.0/src/gtest-all.cc"] + ), + hdrs = glob([ + "googletest-release-1.7.0/include/**/*.h", + "googletest-release-1.7.0/src/*.h" + ]), + copts = [ + "-Iexternal/gtest/googletest-release-1.7.0/include" + ], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], +) +``` + +This is somewhat messy: everything is prefixed with googletest-release-1.7.0 as a byproduct +of the archive's structure. You can make `new_http_archive` strip this prefix by +adding the `strip_prefix` attribute: + +```python +new_http_archive( + name = "gtest", + url = "https://github.com/google/googletest/archive/release-1.7.0.zip", + sha256 = "b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0", + build_file = "gtest.BUILD", + strip_prefix = "googletest-release-1.7.0", +) +``` + +Then `gtest.BUILD` would look like this: + +```python +cc_library( + name = "main", + srcs = glob( + ["src/*.cc"], + exclude = ["src/gtest-all.cc"] + ), + hdrs = glob([ + "include/**/*.h", + "src/*.h" + ]), + copts = ["-Iexternal/gtest/include"], + linkopts = ["-pthread"], + visibility = ["//visibility:public"], +) +``` + +Now `cc_` rules can depend on `@gtest//:main`. + +## Writing and running C++ tests + +For example, we could create a test `./test/hello-test.cc` such as: + +```cpp +#include "gtest/gtest.h" +#include "lib/hello-greet.h" + +TEST(HelloTest, GetGreet) { + EXPECT_EQ(get_greet("Bazel"), "Hello Bazel"); +} +``` + +Then create `./test/BUILD` file for your tests: + +```python +cc_test( + name = "hello-test", + srcs = ["hello-test.cc"], + copts = ["-Iexternal/gtest/include"], + deps = [ + "@gtest//:main", + "//lib:hello-greet", + ], +) +``` + +Note in order to make `hello-greet` visible to `hello-test`, we have to add `"//test:__pkg__",` to `visibility` attribute in `./lib/BUILD`. + +Now you can use `bazel test` to run the test. + +{% highlight bash %} +bazel test test:hello-test +{% endhighlight %} + +This produces the following output: + +{% highlight bash %} +INFO: Found 1 test target... +Target //test:hello-test up-to-date: + bazel-bin/test/hello-test +INFO: Elapsed time: 4.497s, Critical Path: 2.53s +//test:hello-test PASSED in 0.3s + +Executed 1 out of 1 tests: 1 test passes. +{% endhighlight %} + + +## Adding dependencies on precompiled libraries + +If you want to use a library that you only have a compiled version of (e.g., +headers and a .so) wrap it in a `cc_library` rule: + +```python +cc_library( + name = "mylib", + srcs = ["mylib.so"], + hdrs = ["mylib.h"], +) +``` + +Then other C++ targets in your workspace can depend on this rule. diff --git a/site/docs/tutorial/environment.md b/site/docs/tutorial/environment.md index 8ce35d68a2..68548dfdfc 100644 --- a/site/docs/tutorial/environment.md +++ b/site/docs/tutorial/environment.md @@ -1,4 +1,91 @@ --- -layout: redirect -redirect: docs/tutorial/environment.html +layout: documentation +title: Tutorial - Set Up Your Environment --- + +# Tutorial - Set Up Your Environment + +The first step in this tutorial is to set up your environment. + +Here, you'll do the following: + +* Install Bazel +* Install Android Studio and the Android SDK +* Install Xcode (macOS (OS X) only) +* Get the sample project from the GitHub repo + +## Install Bazel + +Follow the [installation instructions](/docs/install.md) to install Bazel and +its dependencies. + +## Install the Android SDK tools + +Do the following: + +1. Download and install the + [Android SDK Tools](https://developer.android.com/sdk/index.html#Other). + +2. Run the Android SDK Manager and install the following packages: + + <table class="table table-condensed table-striped"> + <thead> + <tr> + <td>Package</td> + <td>SDK directory</td> + </tr> + </thead> + <tbody> + <tr> + <td>Android SDK Platform Tools</td> + <td><code>platform-tools</code></td> + </tr> + <tr> + <td>Android SDK Build Tools</td> + <td><code>build-tools</code></td> + </tr> + <tr> + <td>Android SDK Platform</td> + <td><code>platform</code></td> + </tr> + </tbody> + </table> + + The SDK Manager is an executable named `android` located in the `tools` + directory. + +## Install Xcode (OS X only) + +If you are following the steps in this tutorial on Mac OS X, download and +install [Xcode](https://developer.apple.com/xcode/downloads/). The Xcode +download contains the iOS libraries, Objective-C compiler other tools +required by Bazel to build the iOS app. + +## Get the sample project + +You also need to get the sample project for the tutorial from GitHub: + +[https://github.com/bazelbuild/examples/](https://github.com/bazelbuild/examples/) + +The GitHub repo has two branches: `source-only` and `master`. The `source-only` +branch contains the source files for the project only. You'll use the files in +this branch in this tutorial. The `master` branch contains both the source files +and completed Bazel `WORKSPACE` and `BUILD` files. You can use the files in this +branch to check your work when you've completed the tutorial steps. + +Enter the following at the command line to get the files in the `source-only` +branch: + +```bash +cd $HOME +git clone -b source-only https://github.com/bazelbuild/examples +``` + +The `git clone` command creates a directory named `$HOME/examples/`. This +directory contains several sample projects for Bazel. The project files for this +tutorial are in `$HOME/examples/tutorial`. + +## What's next + +Now that you have set up your environment, you can +[set up a Bazel workspace](workspace.md). diff --git a/site/docs/tutorial/index.md b/site/docs/tutorial/index.md deleted file mode 100644 index 3d2efada3e..0000000000 --- a/site/docs/tutorial/index.md +++ /dev/null @@ -1,4 +0,0 @@ ---- -layout: redirect -redirect: docs/tutorial/app.html ----
\ No newline at end of file diff --git a/site/docs/tutorial/ios-app.md b/site/docs/tutorial/ios-app.md index e77b36907e..abf49d27c4 100644 --- a/site/docs/tutorial/ios-app.md +++ b/site/docs/tutorial/ios-app.md @@ -1,4 +1,207 @@ --- -layout: redirect -redirect: docs/tutorial/ios-app.html +layout: documentation +title: Tutorial - Build an iOS App --- + +# Tutorial - Build an iOS App + +Like the [Android app](android-app.md) you built in the previous step, the iOS +app is a simple mobile app that communicates with the +[backend server](backend-server.md). + +Here, you'll do the following: + +* Review the source files for the app +* Create a `BUILD` file +* Build the app for the simulator +* Find the build outputs +* Run/Debug the app on the simulator +* Build the app for a device +* Install the app on a device + +Note that, unlike with the Android app, you don't have to modify your +`WORKSPACE` file to add iOS-specific external dependencies. + +If you're following the steps in this tutorial on macOS (OS X), you can go ahead +and build the sample iOS app as described below. If you are on Linux, skip ahead +to the [next step](backend-server.md). + +## Review the source files + +Let's take a look at the source files for the app. These are located in +`$WORKSPACE/ios-app/UrlGet`. Again, you're just looking at these files now to +become familiar with the structure of the app. You don't have to edit any of the +source files to complete this tutorial. + +## Create a BUILD file + +At a command-line prompt, open your new `BUILD` file for editing: + +```bash +vi $WORKSPACE/ios-app/BUILD +``` + +## Add an objc_library rule + +Bazel provides several build rules that you can use to build an app for the +iOS platform. For this tutorial, you'll first use the +[`objc_library`](/docs/be/objective-c.html#objc_library) rule to tell Bazel +how to build an +[static library](https://developer.apple.com/library/ios/technotes/iOSStaticLibraries/Introduction.html) +from the app source code and Xib files. Then you'll use the +`objc_binary` rule to tell it how to bundle the iOS application. (Note that +this is a minimal use case of the Objective-C rules in Bazel. For example, you +have to use the `ios_application` rule to build multi-architecture iOS +apps.) + +Add the following to your `BUILD` file: + +```python +objc_library( + name = "UrlGetClasses", + srcs = [ + "UrlGet/AppDelegate.m", + "UrlGet/UrlGetViewController.m", + ], + hdrs = glob(["UrlGet/*.h"]), + xibs = ["UrlGet/UrlGetViewController.xib"], +) +``` + +Note the name of the rule, `UrlGetClasses`. + +## Add an objc_binary rule + +The [`objc_binary`](/docs/be/objective-c.html#objc_binary) rule creates a +binary to be bundled in the application. + +Add the following to your `BUILD` file: + +```python +objc_binary( + name = "ios-app-binary", + srcs = [ + "UrlGet/main.m", + ], + deps = [ + ":UrlGetClasses", + ], +) + +``` +Note how the `deps` attribute references the output of the +`UrlGetClasses` rule you added to the `BUILD` file above. + +## Add an ios_application rule + +The [`ios_application`](/docs/be/objective-c.html#ios_application) rule +creates the bundled `.ipa` archive file for the application and also generates +an Xcode project file. + +Add the following to your `BUILD` file: + +```python +ios_application( + name = "ios-app", + binary = ":ios-app-binary", + infoplist = "UrlGet/UrlGet-Info.plist", +) +``` + +Now, save and close the file. You can compare your `BUILD` file to the +[completed example](https://github.com/bazelbuild/examples/blob/master/tutorial/ios-app/BUILD) +in the `master` branch of the GitHub repo. + +## Build the app for the simulator + +Make sure that your current working directory is inside your Bazel workspace: + +```bash +cd $WORKSPACE +``` + +Now, enter the following to build the sample app: + +```bash +bazel build //ios-app:ios-app +``` + +Bazel now launches and builds the sample app. During the build process, its +output will appear similar to the following: + +```bash +INFO: Found 1 target... +Target //ios-app:ios-app up-to-date: + bazel-bin/ios-app/ios-app.ipa + bazel-bin/ios-app/ios-app.xcodeproj/project.pbxproj +INFO: Elapsed time: 3.765s, Critical Path: 3.44s +``` + +## Find the build outputs + +The `.ipa` file and other outputs are located in the +`$WORKSPACE/bazel-bin/ios-app` directory. + +## Run/Debug the app on the simulator + +You can now run the app from Xcode using the iOS Simulator. To run the app, +open the project directory `$WORKSPACE/bazel-bin/ios-app/ios-app.xcodeproj` in +Xcode, choose an iOS Simulator as the runtime scheme and then click the **Run** +button. + +**Note:** If you change anything about the project file set in Xcode (for +example, if you add or remove a file, or add or change a dependency), you must +rebuild the app using Bazel and then re-open the project. + +## Build the app for a device + +You need to set up bazel so that it can find the appropriate provisioning +profile for the device you want to build for. To set up the "default" +provisioning profile for all bazel builds: + + 1. Go to [Apple Profiles](https://developer.apple.com/account/ios/profile/profileList.action) + and download the appropriate provisioning profile for your device. + If this is confusing, please refer to [Apple's documentation](https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/MaintainingProfiles/MaintainingProfiles.html). + 1. Move your profile into `$WORKSPACE/tools/objc`. + 1. Optional - You may want to add your profile to your `.gitignore`. + 1. Edit `$WORKSPACE/tools/objc/BUILD` and add: + + ```python + filegroup( + name = "default_provisioning_profile", + srcs = ["<NAME OF YOUR PROFILE>.mobileprovision"], + ) + ``` + +Now you should be able to build the app for your device: + +```bash +bazel build //ios-app:ios-app --ios_multi_cpus=armv7,arm64 +``` + +This will build the app "fat". If you would prefer just to build for +your specific device architecture you can designate a single architecture. + +If you would like to select a specific Xcode version you can do so +with the `--xcode_version=7.2` option. If for some reason you need to specify +a specific SDK version you can use the `--ios_sdk_version=9.2` option, but the +`--xcode_version` should be sufficient in most circumstances. + +If you would like to specify a minimum version of iOS to run against, you can +do so with the `--ios_minimum_os=7.0` option. + +## Install the app on a device + +The easiest way to install the app on the device is to launch Xcode and use the +`Windows > Devices` command. Select your plugged in device from the list on the +left, and then add the app by clicking on the "plus" sign under installed apps +and selecting the `.ipa` that you built. + +If your app does not launch, please make sure that your device was on your +provisioning profile. The `View Device Logs` button on the `Devices` screen in +Xcode may provide other information as to what has gone wrong. + +## What's next + +The next step is to build a [backend server](backend-server.md) for the two +mobile apps you built in this tutorial. diff --git a/site/docs/tutorial/java.md b/site/docs/tutorial/java.md index aeeac5483a..a916aeb5be 100644 --- a/site/docs/tutorial/java.md +++ b/site/docs/tutorial/java.md @@ -1,4 +1,576 @@ --- -layout: redirect -redirect: docs/tutorial/java.html +layout: documentation +title: Introduction to Bazel --- + +Introduction to Bazel: Build Java +========== + +This tutorial is an introduction for anyone getting started with Bazel. It +focuses on the concepts, setup, and use of Bazel using a Java sample project. + +Estimated time: 30 min + +## What you will learn + +In this tutorial you'll learn how to: + +* Build a target from source files +* Produce a visual representation of the dependency graph +* Break a monolithic binary into smaller libraries +* Use multiple Bazel packages +* Control the visibility of a target between packages +* Use labels to reference a target +* Deploy your target + +## Before you begin + +* [Install Bazel](/docs/install.md) + +## Create the sample Java project + +The first step in this tutorial is to create a small Java project. Even though +the project is in Java, this tutorial will focus on concepts that are helpful +for using Bazel in any language. + +1. Create the directory `~/my-project/` + +2. Move to this directory: + + ``` + cd ~/my-project + ``` + +3. Create the following directories under `my-project`: + + ``` + mkdir -p src/main/java/com/example + ``` + + Note that path uses conventions specific to Java programs. Programs written + in other languages may have a different workspace path and directory + structure. + +4. In the directory you created, add a file called `Greeting.java` with the + following contents: + + ```java + package com.example; + + public class Greeting { + public static void sayHi() { + System.out.println("Hi!"); + } + } + ``` + +5. Add a second file `ProjectRunner.java` with the following contents: + + ```java + package com.example; + + public class ProjectRunner { + public static void main(String args[]) { + Greeting.sayHi(); + } + } + ``` + +You’ve now created a small Java project. It contains one file that will be +compiled into a library, and another which will be an executable that uses the +library. + +The rest of this tutorial focuses on setting up and using Bazel to build these +source files. + +## Build with Bazel + +### Set up the workspace + +Workspaces are directories that contain the source files for one or more +software projects, as well as a WORKSPACE file and BUILD files that contain +the instructions that Bazel uses to build the software. The workspace may also +contain symbolic links to output directories. + +To define the workspace, create an empty text file at the root of the project +and name it `WORKSPACE`. You now have: `~/my-project/WORKSPACE`. + +This directory and its subdirectories are now part of the same workspace. When +Bazel builds an output, all inputs and dependencies must be in the same +workspace. Anything in different workspaces are independent of each other, +though there are ways to link workspaces that are beyond the scope of this +introduction tutorial. + +If you also do the [C++ tutorial](/docs/tutorial/cpp.md), you’ll notice it uses +the same workspace. Bazel can understand multiple targets in multiple languages +in a single workspace. + +### Create a BUILD file + +Bazel looks for files named `BUILD` which describe how to build the project. + +1. In the `~/my-project` directory, create a file and name it BUILD. This BUILD + file is a sibling of the WORKSPACE file. + + In the BUILD file, you use a declarative language similar to Python to + create instances of Bazel rules. These instances are called *rule targets*. + In Bazel, *targets* are either files or rule targets and they are the + elements in a workspace that you can ask Bazel to build. + + For this project, you’ll use the built-in rule `java_binary`. Bazel's + built-in rules are all documented in the + [Build Encyclopedia](/docs/be/overview.html). You can also create your own + rules using the [Bazel rule extension framework](/docs/skylark/concepts.md). + +2. Add this text to the BUILD file: + + ``` + java_binary( + name = "my-runner", + srcs = glob(["src/main/java/com/example/*.java"]), + main_class = "com.example.ProjectRunner", + ) + ``` +As you can see, the text in the BUILD file doesn’t describe what Bazel does +when it executes this rule target. The rule’s implementation handles the +complexity of how it works (such as the compiler used). + +You can treat the rule as a black box, focusing on what inputs it needs, and +the outputs it produces. This rule builds a Java archive ("jar file") as well +as a wrapper shell script with the same name as the rule target. + +When you’re writing your own BUILD file, go to the +[Build Encyclopedia](/docs/be/overview.html) for a description of what a rule +does and for its list of possible attributes you can define. For example, +here’s the entry for the [java_binary](/docs/be/java.html#java_binary) rule in +the Build Encyclopedia. The Build Encyclopedia has information about all of the +rules that are compiled into Bazel. + +Let’s take a look at the rule target that you added to the BUILD file. + +Each rule instantiation in the BUILD file creates one rule target. Here, you’ve +instantiated the rule `java_binary`, creating the target `my-runner`. + +Different rules will require different attributes, though all must include a +“name” attribute. You use these attributes to explicitly list all of the +target’s dependencies and options. In the target above: + +* `my-runner` is the name of the rule target created + +* `glob(["src/main/java/com/example/*.java"])` includes every file in that + directory that ends with .java (see the Build Encyclopedia for more + information about [globbing](/docs/be/functions.html#glob)) + +* `"com.example.ProjectRunner"` specifies the class that contains the main + method. + +### Build with Bazel + +Now you’re ready to build the Java binary. To do so, you’ll use the command +`bazel build` with the target label `//:my-runner`. You reference targets by +using their label. Label syntax is described later in this tutorial. + +1. Build my-runner by using this command: + + ``` + bazel build //:my-runner + ``` + + You’ll see output similar to: + + ``` + INFO: Found 1 target... + Target //:my-runner up-to-date: + bazel-bin/my-runner.jar + bazel-bin/my-runner + INFO: Elapsed time: 1.021s, Critical Path: 0.83s + ``` + +2. Now execute the file by using this command: + + ``` + bazel-bin/my-runner + ``` + +Congratulations, you've built your first Bazel target! + +Let’s take a look at what you built. In `~/my-project`, Bazel created the +directory `bazel-bin` as well as other directories to store information about +the build. Open this directory to look at the files created during the build +process. These output directories keep the outputs separate from your source +tree. + +### Review the dependency graph + +Bazel requires build dependencies to be explicitly declared in BUILD +files. The build will fail if dependencies are missing, so when a build works +the declared dependencies are accurate. With this explicit information about +dependencies, Bazel creates a build graph and uses it to accurately perform +incremental builds. Our small Java project isn’t too exciting, but let’s check +out its build graph. + +The command `bazel query` retrieves information about the graph and the +relationships between targets. Let’s use it to produce a visual representation +of the build graph. + +1. From the root of the workspace (`my-project`), produce a text description + of the graph by using the command: + + ``` + bazel query --noimplicit_deps 'deps(//:my-runner)' --output graph + ``` + +2. Then, paste the output into Graphviz + ([http://www.webgraphviz.com/](http://www.webgraphviz.com/)) to see the + visual representation. + + The graph for the target my-runner will look like this: + +  + +You can see that `my-runner` depends on the two source files in your Java +project. + +You have now set up the workspace and BUILD file, and used Bazel to build your +project. You have also created a visual representation of the build graph to +see the structure of your build. + +## Refine your Bazel build + +### Add dependencies + +Creating one rule target to build your entire project may be sufficient for +small projects. As projects get larger it's important to break up the build +into self-contained libraries that can be assembled into a final product. +Self-contained libraries mean that everything doesn't need to be rebuilt after +small changes and that Bazel can parallelize more of the build steps. These +self-contained libraries also encourages good code hygiene. + +To break up a project, create a separate rule target for the each subcomponent +and then add the subcomponents as dependencies. For the project in this +tutorial, create a rule target to compile the library, and make the executable +depend on it. + +1. Replace the text in the BUILD file with the text below: + + ``` + java_binary( + name = "my-runner", + srcs = ["src/main/java/com/example/ProjectRunner.java"], + main_class = "com.example.ProjectRunner", + deps = [":greeter"], + ) + + java_library( + name = "greeter", + srcs = ["src/main/java/com/example/Greeting.java"], + ) + ``` + +The new `deps` attribute in `java_binary` tells Bazel that the `greeter` library +will be needed to compile the binary. Rules for many languages support the +`deps` attribute, though the exact semantics of the attribute will vary based +on the language and the type of target. The rule +[java_library](/docs/be/java.html#java_library) compiles sources into +a .jar file. Remember to go to the [Build Encyclopedia](/docs/be/overview.html) +for details about specific rules. + +This BUILD file builds the same files as before, but in a different way: now +Bazel will first build the `greeter` library and then build `my-runner`. + +2. Try building //:my-runner using the command: + + ``` + bazel build //:my-runner + ``` + + You’ll see output similar to: + + ``` + INFO: Found 1 target... + Target //:my-runner up-to-date: + bazel-bin/my-runner.jar + bazel-bin/my-runner + INFO: Elapsed time: 2.454s, Critical Path: 1.58s + ``` + + 3. Execute the file by using this command:: + + ``` + bazel-bin/my-runner + ``` + +If you now edit `ProjectRunner.java` and rebuild `my-runner`, the source file +`Greeting.java` will not be recompiled. When the BUILD file had only the one +target, both source files would be recompiled after any change. + +Looking at the dependency graph, you can see that `my-runner` depends on the +same inputs as it did before, but the structure of the build is different. + +The original dependency graph for `my-runner` looked link this: + + + +The dependency graph for `my-runner` after adding a dependency looks like this: + + + +### Use multiple packages + +For larger projects, you will often be dealing with several directories in your +workspace. You can organize your build process by adding a BUILD file to the +top directory of source files that you want to organize together. A directory +containing a BUILD file is called a package. + +Note that Bazel and Java both have the concept of a package. These are +unrelated to each other, though both are related to the structure of the +directories. + +Let’s build the java project using multiple packages. + +1. First, let’s make the Java project a bit more complex. + + 1. Add the following directory and file: + + ``` + mkdir -p src/main/java/com/example/cmdline + ``` + 2. In the directory cmdline, add the file Runner.java with the following + contents: + + ```java + package com.example.cmdline; + + import com.example.Greeting; + + public class Runner { + public static void main(String args[]) { + Greeting.sayHi(); + } + } + ``` + + Now you have a slightly larger Java project that you can organize with + multiple packages. + +2. In the directory `src/main/java/com/example/cmdline`, add an empty text + file and name it BUILD. The structure of the Java project is now: + + ``` + ├── BUILD + ├── src + │ └── main + │ └── java + │ └── com + │ └── example + │ ├── cmdline + │ │ ├── BUILD + │ │ └── Runner.java + │ ├── Greeting.java + │ └── ProjectRunner.java + └── WORKSPACE + ``` + + Each directory in the workspace can be part of only one package. The + workspace now has two BUILD files, and so has two packages: + + 1. The directory `my-project` and its subdirectories (but not including + subdirectories with their own BUILD file, such as `cmdline`), and + + 2. The directory `cmdline` and any subdirectories. + +3. In the new BUILD file, add the following text: + + ``` + java_binary( + name = "runner", + srcs = ["Runner.java"], + main_class = "com.example.cmdline.Runner", + deps = ["//:greeter"] + ) + ``` + + The file `Runner.java` depends on `com.example.Greeting`. In the BUILD file + this dependency is shown by listing the rule target `greeter` (with the + label `//:greeter`). + + Below is what the dependency graph for runner will look like. You can see + how `//:greeter` gives the dependency on `Greeting.java`. + +  + + +4. However, if you try to build runner right now you'll get a permissions + error. You can see the permission error by trying to build the target using + the command: + + ``` + bazel build //src/main/java/com/example/cmdline:runner + ``` + + By default, rule targets are private, which means that they can only be + depended on by targets in the same BUILD file. This privacy prevents + libraries that are implementation details from leaking into public APIs, + but it also means that you must explicitly allow `runner` to depend on + `//:greeter`. + + +5. Make a rule target visible to rule targets in other BUILD files by adding + a `visibility` attribute. To make the `greeter` rule target in + `~/my-project/BUILD` visible to any rule target in the new package, add the + following visibility attribute: + + ``` + java_library( + name = "greeter", + srcs = ["src/main/java/com/example/Greeting.java"], + visibility = ["//src/main/java/com/example/cmdline:__pkg__"], + ) + ``` + + The target `//:greeter` is now visible to any target in the + `//src/main/java/com/example/cmdline` package. + + See the Build Encyclopedia for more + [visibility options](/docs/be/common-definitions.html#common.visibility). + + +6. Now you can build the runner binary by using the command: + + ``` + bazel build //src/main/java/com/example/cmdline:runner + ``` + + You’ll see output similar to: + + ``` + INFO: Found 1 target... + Target //src/main/java/com/example/cmdline:runner up-to-date: + bazel-bin/src/main/java/com/example/cmdline/runner.jar + bazel-bin/src/main/java/com/example/cmdline/runner + INFO: Elapsed time: 1.576s, Critical Path: 0.81s + ``` + + +7. Execute the file by using this command: + + ``` + bazel-bin/src/main/java/com/example/cmdline/runner + ``` + +You’ve now refined your build so that it is broken down into smaller +self-contained libraries, and so that the explicit dependencies are more +granular. You’ve also built the Java project using multiple packages. + +## Use labels to reference targets + +In the BUILD files and in the command line, you have been using target labels +to reference targets. The label’s syntax is: `//path/to/package:target-name`, +where “`//`” is the workspace’s root, and “`:`” separates the package name and +the target name. If the target is a rule target and so defined in a BUILD file, +“`path/to/package`” would be the path of the BUILD file itself. “`Target-name`” +would be the same as the “`name`” attribute in the target in the BUILD file. + +The first BUILD file you created in this tutorial is in the same directory as +the WORKSPACE file. When referencing rule targets defined in that file, nothing +is needed for the path to the package because the workspace root and the package +root are the same directory. Here are the labels of the two targets defined +in that first BUILD file: + +``` +//:my-runner +//:greeter +``` + +The second BUILD file has a longer path from the workspace root to the package +root. The label for the target in that BUILD file is: + +``` +//src/main/java/com/example/cmdline:runner +``` + +Target labels can be shortened in a variety of ways. Within a BUILD file, if +you’re referencing a target from the same package, you can write the label +starting at “`:`”. For example, the rule target `greeter` can always be written +as `//:greeter`, and in the BUILD file where it’s defined, it can also be +written as `:greeter`. This shortened label in a BUILD file makes it immediately +clear which targets are in the current package. + +A rule target’s name will always be defined by its name attribute. A target’s +name is a bit more complex when it’s in a directory other than the root +of the package. In that case, the target’s label is: +`//path/to/package:path/to/file/file_name`. + +## Package a Java target for deployment + +To understand what you’ve built and what else can be built with Bazel, you need +to understand the capabilities of the rules used in your BUILD files. Always go +to the [Build Encyclopedia](/docs/be/overview.html) for this information. + +Let’s look at packaging a Java target for deployment, which requires you to +know the capabilities of the rule `java_binary`. + +You’re able to run the Java binaries you created in this tutorial, but you +can’t simply run it on a server, because it relies on the greeting library jar +to actually run. "Packaging an artifact so it can be run reliably outside the +development environment involves bundling it with all of its runtime +dependencies. Let's see now what’s needed to package the binaries. + +The rule [java_binary](/docs/be/java.html#java_binary) produces a Java archive +(“jar file”) and a wrapper shell script. The file `<target-name>_deploy.jar` +is suitable for deployment, but it’s only built by this rule if explicitly +requested. Let’s investigate. + +1. Look at the contents of the output `runner.jar` by using this command: + + ``` + jar tf bazel-bin/src/main/java/com/example/cmdline/runner.jar + ``` + + You’ll see output similar to: + + ``` + META-INF/ + META-INF/MANIFEST.MF + com/ + com/example/ + com/example/cmdline/ + com/example/cmdline/Runner.class + ``` + + You can see that `runner.jar` contains `Runner.class`, but not its + dependency `Greeting.class`. The `runner` script that Bazel generates adds + the greeter jar to the classpath, so running this program works locally. It + will not work if you want to copy `runner.jar` to another machine and use + it as a standalone binary. + + +2. The rule `java_binary` allows you to build a self-contained binary that can + be deployed. To create this binary, build `runner_deploy.jar` (or, more + generally, `<target-name>_deploy.jar`) by using this command: + + ``` + bazel build //src/main/java/com/example/cmdline:runner_deploy.jar + ``` + + You’ll see output similar to: + + ``` + INFO: Found 1 target... + Target //src/main/java/com/example/cmdline:runner_deploy.jar up-to-date: + bazel-bin/src/main/java/com/example/cmdline/runner_deploy.jar + INFO: Elapsed time: 1.700s, Critical Path: 0.23s + ``` + + The file runner_deploy.jar will contain all of its dependencies, and so can + be used as a standalone binary. + +You’ve now created a Java target that you can distribute and deploy. To do so, +you had to be aware of what outputs the Bazel Java rule `java_binary` is able to +produce. + +## Further topics + +* Try the tutorial [Build C++](/docs/tutorial/cpp.md). +* Try the tutorial [Build Mobile Application](/docs/tutorial/app.md). + diff --git a/site/docs/tutorial/review.md b/site/docs/tutorial/review.md index 16557f7527..62d4501ad4 100644 --- a/site/docs/tutorial/review.md +++ b/site/docs/tutorial/review.md @@ -1,4 +1,29 @@ --- -layout: redirect -redirect: docs/tutorial/review.html +layout: documentation +title: Tutorial - Review --- + +# Tutorial - Review + +In this tutorial, you used Bazel to build an [Android app](android-app.md), +an [iOS app](ios-app.md) and a [backend server](backend-server.md) that runs on +Google App Engine. + +To build these software outputs, you: + +* Set up a Bazel [workspace](workspace.md) that contained the source code + for the components and a `WORKSPACE` that identifies the top level of the + workspace directory +* Created a `BUILD` file for each component +* Updated the `WORKSPACE` file to contain references to the required + external dependencies +* Ran Bazel to build the software components + +The built mobile apps and backend server application files are located in the +`$WORKSPACE/bazel-bin` directory. + +Note that completed `WORKSPACE` and `BUILD` files for this tutorial are located +in the +[master branch](https://github.com/bazelbuild/examples/tree/master/tutorial) +of the GitHub repo. You can compare your work to the completed files for +additional help or troubleshooting. diff --git a/site/docs/tutorial/workspace.md b/site/docs/tutorial/workspace.md index 76a2a7cd14..eee1b1a350 100644 --- a/site/docs/tutorial/workspace.md +++ b/site/docs/tutorial/workspace.md @@ -1,4 +1,53 @@ --- -layout: redirect -redirect: docs/tutorial/workspace.html +layout: documentation +title: Tutorial - Set Up a Workspace --- + +# Tutorial - Set Up a Workspace + +A [workspace](/docs/build-ref.html#workspaces) is a directory that contains the +source files for one or more software projects, as well as a `WORKSPACE` file +and `BUILD` files that contain the instructions that Bazel uses to build +the software. The workspace may also contain symbolic links to output +directories. + +A workspace directory can be located anywhere on your filesystem. In this +tutorial, your workspace directory is `$HOME/examples/tutorial/`, which +contains the sample project files you cloned from the GitHub repo in the +previous step. + +Note that Bazel itself doesn't make any requirements about how you organize +source files in your workspace. The sample source files in this tutorial are +organized according to common conventions for Android apps, iOS apps and App +Engine applications. + +For your convenience, set the `$WORKSPACE` environment variable now to refer to +your workspace directory. At the command line, enter: + +```bash +export WORKSPACE=$HOME/examples/tutorial +``` + +## Create a WORKSPACE file + +Every workspace must have a text file named `WORKSPACE` located in the top-level +workspace directory. This file may be empty or it may contain references +to [external dependencies](/docs/external.html) required to build the +software. + +For now, you'll create an empty `WORKSPACE` file, which simply serves to +identify the workspace directory. In later steps, you'll update the file to add +external dependency information. + +Enter the following at the command line: + +```bash +touch $WORKSPACE/WORKSPACE +``` + +This creates the empty `WORKSPACE` file. + +## What's next + +Now that you've set up your workspace, you can +[build the Android app](android-app.md). diff --git a/site/docs/windows.md b/site/docs/windows.md index 8bf61e0e3b..5a0b6b51c4 100644 --- a/site/docs/windows.md +++ b/site/docs/windows.md @@ -1,4 +1,154 @@ --- -layout: redirect -redirect: docs/windows.html +layout: documentation +title: Windows --- + +# Using Bazel on Windows + +Windows support is experimental. Known issues are [marked with label +"Windows"](https://github.com/bazelbuild/bazel/issues?q=is%3Aissue+is%3Aopen+label%3A%22category%3A+multi-platform+%3E+windows%22) +on github issues. + +We currently support only 64 bit Windows 7 or higher and we compile Bazel as a +msys2 binary. + +## <a name="install"></a>Installation + +See instructions on the [installation page](install-windows.md). + +## <a name="requirements"></a>Requirements + +Before you can compile or run Bazel, you will need to set some environment +variables: + +```bash +export JAVA_HOME="$(ls -d C:/Program\ Files/Java/jdk* | sort | tail -n 1)" +export BAZEL_SH=c:/tools/msys64/usr/bin/bash.exe +``` + +If you run outside of `bash`, ensure that ``msys-2.0.dll`` is in your ``PATH`` +(if you install msys2 to ``c:\tools\msys64``, just add +``c:\tools\msys64\usr\bin`` to ``PATH``). + +If you have another tool that vendors msys2 (such as msysgit), then +``c:\tools\msys64\usr\bin`` must appear in your ``PATH`` *before* entries for +those tools. + +Similarly, if you have [bash on Ubuntu on +Windows](https://msdn.microsoft.com/en-gb/commandline/wsl/about) installed, you +should make sure ``c:\tools\msys64\usr\bin`` appears in ``PATH`` *before* +``c:\windows\system32``, because otherwise Windows' ``bash.exe`` is used before +msys2's. + +Use ``where msys-2.0.dll`` to ensure your ``PATH`` is set up correctly. + +To **run** Bazel (even pre-built binaries), you will need: + +* Java JDK 8 or later +* [msys2 shell](https://msys2.github.io/) (need to be installed at + ``C:\tools\msys64\``). + * We build against version + [20160205](https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-x86_64-20160205.exe/download), + you will need this version in order to run the pre-built + [release Bazel binaries](https://github.com/bazelbuild/bazel/releases). + * You can also use newer versions or the + [latest version](https://sourceforge.net/projects/msys2/files/latest/download?source=files), + but then you will need to compile Bazel from the distribution archive (the + source zip file) so that it's linked against the right version of + ``msys-2.0.dll``. See also the + [known issues](install-compile-source.md#known-issues-when-compiling-from-source). +* Several msys2 packages. Use the ``pacman`` command to install them: + + ``` + pacman -Syuu gcc git curl zip unzip zlib-devel + ``` + +To **compile** Bazel, in addition to the above you will need: + +* [Visual C++ Build Tools](http://landinghub.visualstudio.com/visual-cpp-build-tools) + or the full [Visual C++](https://www.visualstudio.com/) (as part of Visual + Studio; Community Edition is fine) with Windows SDK installed. +* You may need to apply some patches/workarounds, see the + [known issues](install-compile-source.md#known-issues-when-compiling-from-source). + +## <a name="compiling"></a>Compiling Bazel on Windows + +Ensure you have the [requirements](#requirements). + +To build Bazel: + +* Open the msys2 shell. +* Clone the Bazel git repository as normal. +* Set the environment variables (see above) +* Run ``compile.sh`` in Bazel directory. +* If all works fine, bazel will be built at ``output\bazel.exe``. + + +## <a name="using"></a>Using Bazel on Windows + +Bazel now supports building C++, Java and Python targets on Windows. + +### Build C++ + +To build C++ targets, you will need: + +* [Visual Studio](https://www.visualstudio.com/) +<br/>We are using MSVC as the native C++ toolchain, so please ensure you have Visual +Studio installed with the `Visual C++ > Common Tools for Visual C++` and +`Visual C++ > Microsoft Foundation Classes for C++` features. +(which is NOT the default installation type of Visual Studio). +You can set `BAZEL_VS` environment variable to tell Bazel +where Visual Studio is, otherwise Bazel will try to find the latest version installed. +<br/>For example: `export BAZEL_VS="C:/Program Files (x86)/Microsoft Visual Studio 14.0"` + +* [Python](https://www.python.org/downloads/) +<br/>Both Python 2 and Python 3 are supported. +Currently, we use Python wrapper scripts to call the actual MSVC compiler, so +please make sure Python is installed and its location is added into PATH. +It's also a good idea to set `BAZEL_PYTHON` environment variable to tell Bazel +where Python is. +<br/>For example: `export BAZEL_PYTHON=C:/Python27/python.exe` + +Bazel will auto-configure the location of Visual Studio and Python at the first +time you build any target. +If you need to auto-configure again, just run `bazel clean` then build a target. + +If everything is set up, you can build C++ target now! + +```bash +bazel build examples/cpp:hello-world +./bazel-bin/examples/cpp/hello-world.exe +bazel run examples/cpp:hello-world +``` + +However, with Bazel version prior to 0.5.0, MSVC +toolchain is not default on Windows, you should use flag +`--cpu=x64_windows_msvc` to enable it like this: + +```bash +bazel build --cpu=x64_windows_msvc examples/cpp:hello-world +``` + +### Build Java + +Building Java targets works well on Windows, no special configuration is needed. +Just try: + +```bash +bazel build examples/java-native/src/main/java/com/example/myproject:hello-world +./bazel-bin/examples/java-native/src/main/java/com/example/myproject/hello-world +bazel run examples/java-native/src/main/java/com/example/myproject:hello-world +``` + +### Build Python + +On Windows, we build a self-extracting zip file for executable Python targets, you can even use +`python ./bazel-bin/path/to/target` to run it in native Windows command line (cmd.exe). +See more details in this [design doc](/designs/2016/09/05/build-python-on-windows.html). + +```bash +bazel build examples/py_native:bin +./bazel-bin/examples/py_native/bin +python ./bazel-bin/examples/py_native/bin # This works in both msys and cmd.exe +bazel run examples/py_native:bin +``` diff --git a/site/faq.md b/site/faq.md deleted file mode 100644 index 187eb695fa..0000000000 --- a/site/faq.md +++ /dev/null @@ -1,421 +0,0 @@ ---- -layout: default -title: FAQ -nav: faq ---- - -What is Bazel? --------------- - -Bazel is a tool that automates software builds and tests. Supported build tasks -include running compilers and linkers to produce executable programs and -libraries, and assembling deployable packages for Android, iOS and other target -environments. Bazel is similar to other tools like Make, Ant, Gradle, Buck, -Pants and Maven. - -What is special about Bazel? ----------------------------- - -Bazel was designed to fit the way software is developed at Google. It -has the following features: - -* Multi-language support: Bazel supports Java, Objective-C and C++ out - of the box, and can be extended to support arbitrary programming - languages. - -* High-level build language: Projects are described in the BUILD - language, a concise text format that describes a project as sets of - small interconnected libraries, binaries and tests. In contrast, with - tools like Make, you have to describe individual files and compiler - invocations. - -* Multi-platform support: The same tool and the same BUILD files can - be used to build software for different architectures, and even - different platforms. At Google, we use Bazel to build everything from - server applications running on systems in our data centers to client apps - running on mobile phones. - -* Reproducibility: In BUILD files, each library, test and binary must - specify its direct dependencies completely. Bazel uses this - dependency information to know what must be rebuilt when you make - changes to a source file, and which tasks can run in parallel. This - means that all builds are incremental and will always produce the - same result. - -* Scalable: Bazel can handle large builds; at Google, it is common for - a server binary to have 100k source files, and builds where no files - were changed take about ~200ms. - - -Why doesn't Google use ...? ---------------------------- - -* Make, Ninja: These tools give very exact control over what commands - get invoked to build files, but it's up to the user to write rules - that are correct. - - Users interact with Bazel on a higher level. For example, Bazel has - built-in rules for "Java test", "C++ binary", and notions such as - "target platform" and "host platform". These rules have been battle - tested to be foolproof. - -* Ant and Maven: Ant and Maven are primarily geared toward Java, while - Bazel handles multiple languages. Bazel encourages subdividing - codebases in smaller reusable units, and can rebuild only ones that - need rebuilding. This speeds up development when working with larger - codebases. - -* Gradle: Bazel configuration files are much more structured than - Gradle's, letting Bazel understand exactly what each action does. - This allows for more parallelism and better reproducibility. - -* Pants, Buck: Both tools were created and developed by ex-Googlers at - Twitter and Foursquare, and Facebook respectively. They have been modeled - after Bazel, but their feature sets are different, so they aren't viable - alternatives for us. - - -Where did Bazel come from? --------------------------- - -Bazel is a flavor of the tool that Google uses to build its server -software internally. It has expanded to build other software as well, -like mobile apps (iOS, Android) that connect to our servers. - -Did you rewrite your internal tool as open-source? Is it a fork? ----------------------------------------------------------------- - -Bazel shares most of its code with the internal tool and its rules -are used for millions of builds every day. - -Why did Google build Bazel? ---------------------------- - -A long time ago, Google built its software using large, generated -Makefiles. These led to slow and unreliable builds, which began to -interfere with our developers' productivity and the company's -agility. Bazel was a way to solve these problems. - -Does Bazel require a build cluster? ------------------------------------ - -Google's in-house flavor of Bazel does use [build -clusters](http://google-engtools.blogspot.com/2011/09/build-in-cloud-distributing-build-steps.html), -so Bazel does have hooks in the code base to plug in a remote build -cache or a remote execution system. - -The open source Bazel code runs build operations locally. We believe -that this is fast enough for most of our users, but work is underway -to provide [distributed caching](https://github.com/bazelbuild/bazel/issues/904). - - -How does the Google development process work? ----------------------------------------------- - -For our server code base, we use the following development workflow: - -* All our server code is in a single, gigantic version control - system. - -* Everybody builds their software with Bazel. - -* Different teams own different parts of the source tree, and make - their components available as BUILD targets. - -* Branching is primarily used for managing releases, so everybody - develops their software at the head revision. - -Bazel is a cornerstone of this philosophy: since Bazel requires all -dependencies to be fully specified, we can predict which programs and -tests are affected by a change, and vet them before submission. - -More background on the development process at Google can be found on -the [eng tools blog](http://google-engtools.blogspot.com/). - -Why are you opening up Bazel? ------------------------------ - -Building software should be fun and easy. Slow and unpredictable -builds take the fun out of programming. - -Why would I want to use Bazel? ------------------------------- - -* Bazel may give you faster build times because it can recompile only - the files that need to be recompiled. Similarly, it can skip - re-running tests that it knows haven't changed. - -* Bazel produces deterministic results. This eliminates skew - between incremental and clean builds, laptop and CI system, etc. - -* Bazel can build different client and server apps with the same tool - from the same workspace. For example, you can change a client/server - protocol in a single commit, and test that the updated mobile app - works with the updated server, building both with the same tool, - reaping all the aforementioned benefits of Bazel. - -Can I see examples? -------------------- - -Yes. For a simple example, see: - - <https://github.com/bazelbuild/bazel/blob/master/examples/cpp/BUILD> - -The Bazel source code itself provides a more complex example: - - <https://github.com/bazelbuild/bazel/blob/master/src/BUILD> - -What is Bazel best at? ----------------------- - -Bazel shines at building and testing projects with the following properties: - -* Projects with a large codebase -* Projects written in (multiple) compiled languages -* Projects that deploy on multiple platforms -* Projects that have extensive tests - -Where can I run Bazel? ---------------------------------- - -Currently, Linux and macOS (OS X). Porting to other UNIX platforms should be -straightforward, as long as a JDK is available for the platform. - -What about Windows? -------------------- - -Due to its UNIX heritage, porting Bazel to Windows is significant work. For -example, Bazel uses symlinks extensively, which has varying levels of support -across Windows versions. - -As of version [0.3.0](https://github.com/bazelbuild/bazel/releases/tag/0.3.0) -Bazel supports Windows (bootstrapping itself and running builds), and we are -actively working on improving this support. See our -[blog post](https://bazel.build/blog/2016/09/07/bazel-windows.html) for more -information, as well as the [list of open bugs](https://github.com/bazelbuild/bazel/issues?q=is%3Aissue+is%3Aopen+label%3A%22category%3A+multi-platform+%3E+windows%22). - -What should I not use Bazel for? --------------------------------- - -* Bazel tries to be smart about caching. This means that it is not good - for running build operations whose outputs should not be cached. For example, - the following steps should not be run from Bazel: - - * A compilation step that fetches data from the internet. - * A test step that connects to the QA instance of your site. - * A deployment step that changes your site's cloud configuration. - -* Bazel tries to minimize expensive compilation steps. If you are only - using interpreted languages directly, such as JavaScript or Python, - Bazel will likely not interest you. - -How stable is Bazel's feature set? ----------------------------------- - -The core features (C++, Java, and shell rules) have extensive use -inside Google, so they are thoroughly tested and have very little -churn. Similarly, we test new versions of Bazel across hundreds of -thousands of targets every day to find regressions, and we release new -versions multiple times every month. - -In short, except for features marked as experimental, Bazel should Just Work. -Changes to non-experimental rules will be backward compatible. A more detailed -list of feature support statuses can be found in our -[support document](support.html). - -How stable is Bazel as a binary? --------------------------------- - -Inside Google, we make sure that Bazel crashes are very rare. This -should also hold for our open source codebase. - -How can I start using Bazel? ----------------------------- - -See our [getting started document](docs/getting-started.html). - -Doesn't Docker solve the reproducibility problems? --------------------------------------------------- - -With Docker you can easily create sandboxes with fixed OS releases, -for example, Ubuntu 12.04, Fedora 21. This solves the problem of -reproducibility for the system environment -- that is, "which version of -/usr/bin/c++ do I need?" - -Docker does not address reproducibility with regard to changes in the -source code. Running Make with an imperfectly written Makefile inside a -Docker container can still yield unpredictable results. - -Inside Google, we check tools into source control for reproducibility. -In this way, we can vet changes to tools ("upgrade GCC to 4.6.1") with -the same mechanism as changes to base libraries ("fix bounds check in -OpenSSL"). - -Can I build binaries for deployment on Docker? ----------------------------------------------- - -With Bazel, you can build standalone, statically linked binaries in -C/C++, and self-contained jar files for Java. These run with few -dependencies on normal UNIX systems, and as such should be simple to -install inside a Docker container. - -Bazel has conventions for structuring more complex programs, for example, a -Java program that consumes a set of data files, or runs another -program as subprocess. It is possible to package up such environments -as standalone archives, so they can be deployed on different systems, -including Docker images. - -Can I build Docker images with Bazel? -------------------------------------- - -Yes, you can use our -[Docker rules](https://bazel.build/versions/master/docs/be/docker.html) -to build reproducible Docker images. - -Will Bazel make my builds reproducible automatically? ------------------------------------------------------ - -For Java and C++ binaries, yes, assuming you do not change the -toolchain. If you have build steps that involve custom recipes -(for example, executing binaries through a shell script inside a rule), you -will need to take some extra care: - - * Do not use dependencies that were not declared. Sandboxed - execution (--spawn_strategy=sandboxed, only on Linux) can - help find undeclared dependencies. - - * Avoid storing timestamps and user-IDs in generated files. ZIP files and - other archives are especially prone to this. - - * Avoid connecting to the network. Sandboxed execution can help here - too. - - * Avoid processes that use random numbers, in particular, dictionary - traversal is randomized in many programming languages. - -Do you have binary releases? ----------------------------- - -Yes, you can find the latest release binaries -[here](https://github.com/bazelbuild/bazel/releases/latest). Our release -policy is documented [here](http://bazel.build/support.html). - -I use Eclipse/IntelliJ/XCode. How does Bazel interoperate with IDEs? --------------------------------------------------------------------- - -For IntelliJ, check out the [IntelliJ with Bazel plugin](https://ij.bazel.build). - -For XCode, check out [Tulsi](http://tulsi.bazel.build/). - -For Eclipse, check out [E4B plugin](https://github.com/bazelbuild/e4b). - -For other IDEs, check out the [blog -post](https://bazel.build/blog/2016/06/10/ide-support.html) on how these -plugins work. - -I use Jenkins/CircleCI/TravisCI. How does Bazel interoperate with CI systems? ------------------------------------------------------------------------------ - -Bazel returns a non-zero exit code if the build or test invocation -fails, and this should be enough for basic CI integration. Since -Bazel does not need clean builds for correctness, the CI system should not -be configured to clean before starting a build/test run. - -Further details on exit codes are in the [User Manual](docs/bazel-user-manual.html). - -What future features can we expect in Bazel? --------------------------------------------- - -Our initial goal is to work on Google's internal use-cases. This -includes Google's principal languages (C++, Java, Go) and major -platforms (Linux, Android, iOS). For practical reasons, not all of -these are currently open-sourced. For more details see our -[roadmap](roadmap.html). - -What about Python? ------------------- - -It is possible to write Python rules as extensions (see below). See -the following files for an example of generating self-contained zip -files for python: - - <https://github.com/bazelbuild/bazel/blob/master/tools/build_rules/py_rules.bzl>\\ - <https://github.com/bazelbuild/bazel/tree/master/examples/py> - -We have opened up a subset of our internal Python rules, so they -can be used as helper scripts as part of a build. - -Simplistic support for PEX-style binaries is at -[here](https://github.com/bazelbuild/bazel/blob/master/tools/build_rules/py_rules.bzl). - - -What about Go? --------------- - -Bazel supports Go through an [external rule set](https://github.com/bazelbuild/rules_go) - - -Can I use Bazel for my [INSERT LANGUAGE HERE] project? ------------------------------------------------------- - -We have an extension mechanism called Skylark that allows you to add new rules -without recompiling Bazel. - -For documentation: see [here](/docs/skylark/index.html). We have support for -several languages that use that extension mechanism, see our -[build encyclopedia](/docs/be/overview.html) for the full -list of supported languages. - -I need more functionality. Can I add rules that are compiled into Bazel? ------------------------------------------------------------------------- - -If our extension mechanism is insufficient for your use case, email -the mailing list for advice: <bazel-discuss@googlegroups.com>. - -Can I contribute to the Bazel code base? ----------------------------------------- - -See our [contribution guidelines](contributing.html). - -Why isn't all development done in the open? -------------------------------------------- - -We still have to refactor the interfaces between the public code in -Bazel and our internal extensions frequently. This makes it hard to do -much development in the open. See our [governance plan](governance.html) -for more details. - -How do I contact the team? --------------------------- - -We are reachable at <bazel-discuss@googlegroups.com>. - -Where do I report bugs? ------------------------ - -Send an e-mail to <bazel-discuss@googlegroups.com> or file a bug -[on GitHub](https://github.com/bazelbuild/bazel/issues). - - - -What's up with the word "Blaze" in the codebase? ------------------------------------------------- - -This is an internal name for the tool. Please refer to Bazel as -Bazel. - - -Why do other Google projects (Android, Chrome) use other build tools? ---------------------------------------------------------------------- - -Until the first (Alpha) release, Bazel was not available externally, so open -source projects such as Chromium, Android, etc. could not use it. In addition, -the original lack of Windows support was a problem for building Windows -applications, such as Chrome. - - -How do you pronounce "Bazel"? ------------------------------ - -The same way as "basil" (the herb) in US English: "BAY-zel". It rhymes with -"hazel". IPA: /ˈbeɪzˌəl/ diff --git a/site/governance.md b/site/governance.md deleted file mode 100644 index b0deef8ddc..0000000000 --- a/site/governance.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -layout: contribute -title: Governance ---- - -# Governance - -The Bazel project is led by a core group of contributors, initially Googlers. -The group of core contributors is self-managed. - -## Accepting contributions - -Please also see our [contribution guidelines](contributing.html). - -### Policy - -We use the following rules for accepting code contributions. This is written -from the perspective that there is a group of people who cooperatively support -the project (the *core contributors*). In contrast, external contributors are -not actively supporting the project, but just contributing individual changes. -At this time, all core contributors work for Google, but this will hopefully -change over time. - -1. We require all contributors to sign [Google's Contributor License - Agreement](https://cla.developers.google.com/). - -2. We accept well-written, well-tested contributions of rules written in - [Skylark](docs/skylark/concepts.html), in a `contrib/` directory or similar - with clearly documented support policies. - -3. We accept well-written, well-tested bug fixes to built-in rules. - -4. We accept well-written, well-tested feature contributions if a core - contributor assumes support responsibilities, i.e., readily answers support - questions and works on bugs. This includes feature contributions from - external contributors. If there is no core contributor to support a feature, - then we will deprecate and subsequently delete the feature - we will give - three months' notice in such cases. - -5. We will not accept untested changes, except in very rare cases. - -6. We require a pre-commit code review from a core contributor for all changes. - For the time being, we will have to continue making changes across the - internal and external code bases, which will be reviewed internal to Google. - -7. We will roll back changes if they break the internal development processes - of any of the core contributors. - -8. We will move towards an open governance model where multiple parties have - commit access, roll-back rights, and can provide explicit support for - features or rules. - -9. We will work with interested parties to improve existing extension points - and to establish new extension points if they do not run counter to the - internal requirements of any of the core contributors. - -## Are you done open sourcing Bazel? - -Open sourcing Bazel is a work-in-progress. In particular, we're still working -on open sourcing: - -* Many of our unit and integration tests (which should make contributing patches - easier). -* Full IDE integration. - -Beyond code, we'd like to eventually have all code reviews, bug tracking, and -design decisions happen publicly, with the Bazel community involved. We are not -there yet, so some changes will simply appear in the Bazel repository without -clear explanation. Despite this lack of transparency, we want to support -external developers and collaborate. Thus, we are opening up the code, even -though some of the development is still happening internal to Google. Please -let us know if anything seems unclear or unjustified as we transition to an -open model. - -## Are there parts of Bazel that will never be open sourced? - -Yes, some of the code base either integrates with Google-specific technology or -we have been looking for an excuse to get rid of (or is some combination of the -two). These parts of the code base are not available on GitHub and probably -never will be. - -### Contact the core contributors - -<p class="lead"> -Contact the core team at <a href="mailto:bazel-core@googlegroups.com"> -bazel-core@googlegroups.com</a>. -</p> diff --git a/site/index.html b/site/index.html deleted file mode 100644 index 4d98e0543b..0000000000 --- a/site/index.html +++ /dev/null @@ -1,166 +0,0 @@ ---- -layout: home -title: Home ---- - -<div class="hero"> - <div class="container"> - <img src="{{site_root}}images/bazel-logo.png" - class="img-responsive center-block" /> - <p class="lead">{Fast, Correct} - Choose two</p> - </div> -</div> -<div class="hero-bar"> - <div class="container"> - <a class="btn btn-success" - href="{{ "/docs/install.html" | prepend: site_root }}"> - Get Bazel</a> - <a class="btn btn-success" - href="{{ "/docs/getting-started.html" | prepend: site_root }}"> - Get Started</a> - </div> -</div> -<div class="landing-feature-1"> - <div class="container"> - <div class="row"> - <div class="col-sm-12"> - <h1>Correct, reproducible, fast builds for everyone</h1> - <p class="lead">Build software of any size, quickly and reliably, just as engineers do at Google.</p> - <p>Bazel is Google's own build tool, now publicly available in <a href="#beta">Beta</a>. - Bazel has built-in support for building both client and server software, including client - applications for both Android and iOS platforms. It also provides an extensible framework - that you can use to develop your own build rules. - </p> - </div> - </div> - </div> -</div> -<div class="landing-feature-2"> - <div class="container"> - <div class="row"> - <div class="col-sm-12"> - <h1>Why Bazel?</h1> - <p class="lead">Here's what makes Bazel great.</p> - </div> - </div> - <div class="row"> - <div class="col-sm-4"> - <h3>Speed</h3> - <p>With Bazel's optimized dependency analysis, advanced - caching and parallel execution of build actions, you get - fast, clean builds. - </p> - </div> - <div class="col-sm-4"> - <h3>Scalability</h3> - <p>At Google, Bazel is used by tens of - thousands of engineers to build heavy-duty, mission-critical - infrastructure services as well as public web applications. But it's - great for smaller projects, too. - </p> - </div> - <div class="col-sm-4"> - <h3>Flexibility</h3> - <p>A comprehensive, built-in set of rules lets you build software for - a wide variety of languages and platforms right out of the box. - Skylark, Bazel's extensible rule framework, allows you to develop - and share your own build rules for other languages and platforms. - </p> - </div> - </div> - <div class="row"> - <div class="col-sm-4"> - <h3>Correctness</h3> - <p>Bazel examines the content of source files in your - dependency graph, not just timestamps, to determine when to rebuild. - Because incremental builds are virtually always correct, it's not - necessary to run a clean build "just in case" to ensure correctness. - </p> - </div> - <div class="col-sm-4"> - <h3>Reliability</h3> - <p>When you build software with Bazel, you're running the same code - that has been refined and tested over years of use in the - engineering environment at Google. - </p> - </div> - <div class="col-sm-4"> - <h3>Repeatability</h3> - <p>Builds only use input files that are explicitly declared in the - build specification. On Linux, Bazel runs tools in a sandboxed - environment that contain only the minimum necessary files required. - Even tests are run in sandboxes, which ensures predictable, - environment-independent results. - </p> - </div> - </div> - </div> -</div> -<div class="landing-feature-1"> - <div class="container"> - <div class="row"> - <div class="col-sm-12"> - <h1 id="beta">Bazel is in Beta</h1> - <p>Bazel is in Beta. There are still some functional limitations, - but we believe that Bazel is stable enough for production use. - See our <a href="docs/support.html">Support</a> page for information - about feature status and releases. The Bazel team monitors and - responds actively to bug reports from users. - </p> - </div> - </div> - </div> -</div> -<div class="landing-feature-2"> - <div class="container"> - <div class="row"> - <div class="col-sm-12"> - <h1>Get Started</h1> - </div> - <div class="col-sm-4"> - <h3>Install Bazel</h2> - <p>See our <a href="docs/install.html">Installation Guide</a> to learn - how to download and install Bazel. - </p> - </div> - <div class="col-sm-4"> - <h3>Run Bazel</h2> - <p>See our - <a href="docs/getting-started.html">Getting Started Guide</a> to - learn how to write a BUILD file and run Bazel. - </p> - </div> - <div class="col-sm-4"> - <h3>Bazel Commands</h2> - <p>See our <a href="docs/bazel-user-manual.html">User Manual</a> to - learn about the Bazel command-line tool. - </p> - </div> - </div> - <div class="row"> - <div class="col-sm-12"> - <h3>Learn More</h2> - <p>Take a look at our <a href="docs/install.html">documentation</a> - and read our <a href="faq.html">FAQ</a>. - </p> - </div> - </div> - </div> -</div> - -<div class="landing-feature-1"> - <div class="container"> - <div class="row"> - <div class="col-sm-12"> - <h1>About Bazel</h1> - <p> - <a href="contributing.html">Contribute</a><br /> - <a href="governance.html">Governance Plan</a><br /> - <a href="roadmap.html">Roadmap</a><br /> - <a href="docs/support.html">Support</a><br /> - <a href="https://github.com/bazelbuild/bazel/wiki/Bazel-Users">Who's using Bazel</a> - </p> - </div> - </div> - </div> -</div> diff --git a/site/roadmap.md b/site/roadmap.md deleted file mode 100644 index 3c0a220e2a..0000000000 --- a/site/roadmap.md +++ /dev/null @@ -1,186 +0,0 @@ ---- -layout: contribute -title: Roadmap ---- - -# Bazel Feature Roadmap - -This document describes the Bazel team's plans for introducing features that -will be incorporated into version 1.0. Note that this roadmap only includes -features that the Bazel team itself intends to support. We anticipate that a -number of other features will be added by code contributors. - -For the alpha and beta releases, the Bazel team will maintain two code -repositories: - -* A Google-internal repository, containing both Bazel code and - Google-specific extensions and features -* An external [GitHub repository](https://github.com/bazelbuild/bazel), - containing only the Bazel code. - -We anticipate making the external repository *primary* in the future, that is, -code from Google and non-Google contributors will be committed and tested in the -external repository first, then imported into the internal repository. For -the alpha and beta releases, however, the internal repository will be primary. -Changes to Bazel code will be frequently pushed from the internal to -the external repository. - -## Feature list - -In the following table, each feature is associated with a corresponding -milestone. The convention for the priorities are: - -* P0 feature will block the milestone; we will delay the milestone date - until the feature is shipped. -* P1 feature can delay the milestone if the feature can be shipped with a - reasonable delay (2 months max). -* P2 feature will be dropped and rescheduled for later rather than delaying - the milestone. - -We will update this list when reaching each milestone; some milestones may also -be refined if appropriate. - -<table class="table table-condensed table-bordered"> - <colgroup> - <col class="roadmap-col-phase"/> - <col class="roadmap-col-milestone"/> - <col class="roadmap-col-date"/> - <col class="roadmap-col-features"/> - </colgroup> - <thead> - <tr> - <th>Phase</th> - <th>Milestone</th> - <th>Target date</th> - <th>Features</th> - </tr> - </thead> - <tbody> - <tr> - <td rowspan="9"><b><a name="alpha"></a>Alpha</b></td> - <td rowspan="9"><b>Alpha</b><br/><span class="label label-default">Released</span></td> - <td rowspan="9"><b>2015‑03‑24</b></td> - <td>Linux & OS X Support</td> - </tr> - <tr><td>C++ (<a href="http://bazel.build/docs/be/c-cpp.html#cc_binary">build</a> and <a href="http://bazel.build/docs/be/c-cpp.html#cc_test">test</a>)</td></tr> - <tr><td>Java (<a href="http://bazel.build/docs/be/java.html#java_binary">build</a> and <a href="http://bazel.build/docs/be/java.html#java_test">test</a>)</td></tr> - <tr><td>Objective-C for iOS (<a href="http://bazel.build/docs/be/objective-c.html#objc_binary">build</a>)</td></tr> - <tr><td>Python (<a href="http://bazel.build/docs/be/python.html#py_binary">build</a>)</td></tr> - <tr><td>iOS applications (<a href="http://bazel.build/docs/be/objective-c.html#ios_application">build</a>)</td></tr> - <tr><td>Skylark extension mechanism (<a href="http://bazel.build/docs/skylark/index.html">build</a>)</td></tr> - <tr><td>Basic test suite on GitHub</td></tr> - <tr><td>Support for fetching dependencies from <a href="http://bazel.build/docs/be/workspace.html#maven_jar">Maven repositories</a> - and <a href="http://bazel.build/docs/be/workspace.html#http_archive">web servers</a></td></tr> - <tr> - <td rowspan="44"><b><a name="beta"></a>Beta</b></td> - <td rowspan="9"> - <a href="https://github.com/bazelbuild/bazel/releases/tag/0.1.0"><b>0.1</b> - <br/><span class="label label-default">Released</span></a> - </td> - <td rowspan="9"><b>2015‑09‑01</b></td> - <td>P0. Binary distribution for Linux & OS X</td< - </tr> - <tr><td>P0. Public <a href="http://ci.bazel.build">continuous integration system</a></td></tr> - <tr><td>P0. Support for <a href="http://bazel.build/docs/external.html">fetching transitive dependencies from Maven Central</a></td></tr> - <tr><td>P0. Android application (<a href="http://bazel.build/docs/be/android.html#android_binary">build</a> - and <a href="http://bazel.build/docs/bazel-user-manual.html#mobile-install">install</a>)</td></tr> - <tr><td>P1. Support for <a href="http://bazel.build/docs/external.html">prefetching and caching remote dependencies</a></td></tr> - <tr><td>P1. Docker (<a href="http://bazel.build/docs/be/docker.html">build and load</a>)</td></tr> - <tr><td>P2. <a href="http://bazel.build/docs/bazel-user-manual.html#sandboxing">Sandboxing of actions for Linux</a></td></tr> - <tr><td>P2. AppEngine (<a href="http://bazel.build/docs/be/appengine.html">build and load</a>)</td></tr> - <tr><td>P2. <a href="http://bazel.build/blog/2015/07/29/dashboard-dogfood.html">Test result dashboard</a></tr></td> - <tr> - <td rowspan="5"> - <a href="https://github.com/bazelbuild/bazel/releases/tag/0.2.0"><b>0.2</b> - <br/><span class="label label-default">Released</span></a> - </td> - <td rowspan="5"><b>2016‑02‑18</b></td> - <td>P0. <a href="https://github.com/bazelbuild/bazel/tree/master/src/test/java/com/google/devtools">Significantly increase test coverage</a></td> - </tr> - <tr><td>P0. Support for fetching <a href="http://bazel.build/docs/external.html">remote</a> <a href="http://bazel.build/docs/be/functions.html#load">Skylark rules</a></td></tr> - <tr><td>P2. <a href="https://github.com/bazelbuild/rules_go">Go language support (build and tests)</a></td></tr> - <tr><td>P2. <a href="https://github.com/bazelbuild/bazel/releases/latest">Debian packages for Bazel</a></td></tr> - <tr><td>P2. <a href="http://braumeister.org/formula/bazel">OS X homebrew recipe for distributing Bazel</a></td></tr> - <tr> - <td rowspan="5"> - <a href="https://github.com/bazelbuild/bazel/releases/tag/0.3.0"><b>0.3</b> - <br/><span class="label label-default">Released</span></a> - </td> - <td rowspan="5"><b>2016‑06‑10</b></td> - <td>P0. <a href="http://bazel.build/docs/windows.html">Bazel can bootstrap itself on Windows without requiring admin privileges</a></td></tr> - </tr> - <tr><td>P1. <a href="http://bazel.build/blog/2016/06/10/ide-support.html">Interface for IDE support</a></td></tr> - <tr><td>P1. IDE support for <a href="http://tulsi.bazel.build">Xcode (stable)</a> and <a href="https://github.com/bazelbuild/e4b">Eclipse (experimental)</a></td></tr> - <tr><td>P1. <a href="https://docs.google.com/document/d/1jKbNXOVp2T1zJD_iRnVr8k5D0xZKgO8blMVDlXOksJg">Custom remote repositories using Skylark</a></td></tr> - <tr><td>P2. <a href="https://github.com/bazelbuild/bazel/commit/79adf59e2973754c8c0415fcab45cd58c7c34697">Prototype for distributed caching of build artifact</a></td></tr> - <tr> - <td rowspan="2"> - <a href="https://github.com/bazelbuild/bazel/releases/tag/0.4.0"><b>0.4</b> - <br/><span class="label label-default">Released</span></a> - </td> - <td rowspan="2"><b>2016‑11‑02</b></td> - <td>P0. <a href="https://github.com/bazelbuild/bazel/commit/490f250b27183a886cf70a5fe9e99d9428141b34">Persistent Java compiler is enabled</a></td> - </tr> - <tr> - <td>P2. <a href="https://github.com/bazelbuild/bazel/commit/7b825b8ea442246aabfa6a5a8962abd70855d0da">Sandboxing of action for OS X</a></td> - </tr> - <tr> - <td rowspan="5"> - <a href="https://github.com/bazelbuild/bazel/releases/tag/0.5.0"><b>0.5</b> - <br/><span class="label label-default">Released</span></a> - </td> - <td rowspan="5"><b>2017‑05‑26</b></td> - <td>P0. Support for building and testing Java, C++ and Python on Windows</td> - </tr> - <tr><td>P1. Initial API for a Build Event Protocol</td></tr> - <tr><td>P1. Support for coverage for Java</td></tr> - <tr><td>P1. Bazel installer optionally bundles the JDK</td></tr> - <tr><td>P2. Repository rules no longer have invalidation issues</td></tr> - <tr> - <td rowspan="4"><b>0.6</b></td> - <td rowspan="4"><b>2017‑06</b></td> - <td>P0. Stable API for Remote execution including platform description</td> - </tr> - <tr><td>P0. List of feature to deprecate until version 1.0 are tracked in a publicly available document</td></tr> - <tr><td>P1. Bazel on Windows does not need to install MSYS</td></tr> - <tr><td>P2. Bazel can load workspace recursively</td></tr> - <tr> - <td rowspan="8"><b>0.7</b></td> - <td rowspan="8"><b>2017‑09</b></td> - <td>P0. Skylark is fully documented; strategy for user-provided Skylark rule documentation</td> - </tr> - <tr><td>P0. Support for Android integration testing</td></tr> - <tr><td>P0. Support for Robolectric test for Android</td></tr> - <tr><td>P1. The Build Event Protocol is stable</td></tr> - <tr><td>P1. Support for coverage can be extended in Skylark and C++</td></tr> - <tr><td>P1. Support for testing Skylark rules</td></tr> - <tr><td>P2. All external repositories can use the local cache</td></tr> - <tr><td>P2. Local caching of build artifacts</td></tr> - <tr> - <td rowspan="4"><b>0.8</b></td> - <td rowspan="4"><b>2017‑12</b></td> - <td>P0. Support for iOS integration testing</td> - </tr> - <tr><td>P1. Bazel can build Android application on Windows</td></tr> - <tr><td>P1. Local caching of external repository is turned on by default and has a deletion strategy</td></tr> - <tr><td>P1. Access to native rules functionality from Skylark (<a href="https://bazel.build/designs/2016/08/04/extensibility-for-native-rules.html">"sandwich"</a>)</td></tr> - <tr> - <td rowspan="2"><b>0.9</b></td> - <td rowspan="2"><b>2018‑03</b></td> - <td>P0. Full Windows support</td> - </tr> - <tr><td>P1. Full test suite is open-sourced</td></tr> - <tr> - <td rowspan="7"><b><a name="stable"></a>Stable</b></td> - <td rowspan="7"><b>1.0</b></td> - <td rowspan="7"><b>2018‑06</b></td> - <td>P0. APIs are versioned</td> - </tr> - <tr><td>P0. Github is primary</td></tr> - <tr><td>P1. Deprecated features are removed</td></tr> - <tr><td>P1. Support policy is defined regarding LTS release</td></tr> - <tr><td>P1. Public review process for design documents</td></tr> - <tr><td>P1. Bazel respects the standard for Debian packaging</td></tr> - <tr><td>P2. Bazel is in the list of debian package for the next stable</td></tr> - </tbody> -</table> diff --git a/site/users.md b/site/users.md deleted file mode 100644 index 8c4593c7bb..0000000000 --- a/site/users.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -layout: contribute -title: Bazel Users ---- - -Find a community curated a list of Bazel users in the [Bazel GitHub wiki](https://github.com/bazelbuild/bazel/wiki/Bazel-Users)
\ No newline at end of file diff --git a/site/versions/master/docs/best-practices.md b/site/versions/master/docs/best-practices.md deleted file mode 100644 index ca360954b9..0000000000 --- a/site/versions/master/docs/best-practices.md +++ /dev/null @@ -1,321 +0,0 @@ ---- -layout: documentation -title: Best practices ---- - -# Best practices for Bazel - -This document assumes that you are familiar with Bazel and provides advice on structuring your -projects to take full advantage of Bazel's features. - -The overall goals are: - -- To use fine-grained dependencies to allow parallelism and incrementality. -- To keep dependencies well-encapsulated. -- To make code well-structured and testable. -- To create a build configuration that is easy to understand and maintain. - -These guidelines are not requirements: few projects will be able to adhere to all of them. As the -man page for lint says, "A special reward will be presented to the first person to produce a real -program that produces no errors with strict checking." However, incorporating as many of these -principles as possible should make a project more readable, less error-prone, and faster to build. - -This document uses the requirement levels described in -[this RFC](https://www.ietf.org/rfc/rfc2119.txt). - -## Contents - -- [General structure](#general-structure) - - [Running builds and tests](#running-builds-and-tests) - - [Third party dependencies](#third-party-dependencies) - - [Depending on binaries](#depending-on-binaries) - - [Versioning](#versioning) - - [.bazelrc](#bazelrc) - - [Packages](#packages) -- [BUILD files](#build-files) - - [BUILD file style guide](#build-file-style-guide) - - [Formatting](#formatting) - - [References to targets in the current package](#references-to-targets-in-the-current-package) - - [Target naming](#target-naming) - - [Visibility](#visibility) - - [Dependencies](#dependencies) - - [Globs](#globs) -- [Skylark](#skylark) - - [Skylark style guide](#skylark-style-guide) - - [Packaging rules](#packaging-rules) - - [Rule choice](#rule-choice) -- [WORKSPACE files](#workspace-files) - - [Repository rules](#repository-rules) - - [Custom BUILD files](#custom-build-files) - - [Skylark repository rules](#skylark-repository-rules) -- [Java](#java) - - [Directory structure](#directory-structure) - - [BUILD files](#build-files) -- [C++](#c) - - [BUILD files](#build-files) - - [Include paths](#include-paths) -- [Protos](#protos) - - [Recommended Code Organization](#recommended-code-organization) - -# General structure - -## Running builds and tests - -A project should always be able to run `bazel build //...` and `bazel test //...` successfully on -its stable branch. Targets that are necessary but do not build under certain circumstances (e.g., -require specific build flags, do not build on a certain platform, require license agreements) -should be tagged as specifically as possible (e.g., "`requires-osx`"). This tagging allows -targets to be filtered at a more fine-grained level than the "manual" tag and allows someone -inspecting the BUILD file to understand what a target's restrictions are. - -## Third party dependencies - -Prefer declaring third party dependencies as remote repositories in the WORKSPACE file. If it's -necessary to check third party dependencies into your repository, put them in a directory called -`third_party/` under your workspace directory. Note that all BUILD files in `third_party/` must -include [license](https://bazel.build/versions/master/docs/be/functions.html#licenses) -declarations. - -## Depending on binaries - -Everything should be built from source whenever possible. Generally this means that, instead of -depending on a library `some-library.so`, you'd create a BUILD file and build `some-library.so` -from its sources, then depend on that target. - -Building from source prevents a build from using a library that was built with incompatible flags -or a different architecture. There are also some features like coverage, static analysis, or -dynamic analysis that will only work on the source. - -## Versioning - -Prefer building all code from head whenever possible. When versions must be used, avoid including -the version in the target name (e.g., `//guava`, not `//guava-20.0`). This naming makes the library -easier to update (only one target needs to be updated). It is also more resilient to diamond -dependency issues: if one library depends on `guava-19.0` and one depends on `guava-20.0`, you -could end up with a library that tries to depend on two different versions. If you created a -misleading alias to point both targets to one guava library, then the BUILD files are misleading. - -## `.bazelrc` - -For project-specific options, use the configuration file `_your-workspace_/tools/bazel.rc`. - -For options that you **do not** want to check into source control, create the configuration file -`_your-workspace_/.bazelrc` and add `.bazelrc` to your `.gitignore`. Note that this file has a -different name than the file above (`bazel.rc` vs `.bazelrc`). - -## Packages - -Every directory that contains buildable files should be a package. If a BUILD file refers to files -in subdirectories (e.g., `srcs = ["a/b/C.java"]`) it is a sign that a BUILD file should be added to -that subdirectory. The longer this structure exists, the more likely circular dependencies will be -inadvertently created, a target's scope will creep, and an increasing number of reverse -dependencies will have to be updated. - -# BUILD files - -## BUILD file style guide - -See the [BUILD file style -guide](https://bazel.build/versions/master/docs/skylark/build-style.html). - -## Formatting - -[Buildifier](https://github.com/bazelbuild/buildifier) should be used to achieve the correct -formatting for BUILD files. Editors should be configured to automatically format BUILD files on -save. Humans should not try to format BUILD files themselves. - -If there is a question as to what the correct formatting is, the answer is "how buildifier formats -it." - -## References to targets in the current package - -Files should be referred to by their paths relative to the package directory (without ever using -up-references, such as `..`). Generated files should be prefixed with "`:`" to indicate that they -are not sources. Source files should not be prefixed with `:`. Rules should be prefixed with `:`. -For example, assuming `x.cc` is a source file: - -```python -cc_library( - name = "lib", - srcs = ["x.cc"], - hdrs = [":gen-header"], -) - -genrule( - name = "gen-header", - srcs = [], - outs = ["x.h"], - cmd = "echo 'int x();' > $@", -) -``` - -## Target naming - -Target names should be descriptive. If a target contains one source file, the target should -generally be named after that source (e.g., a `cc_library` for `chat.cc` should be named "`chat`"). - -The eponymous target for a package (the target with the same name as the containing directory) -should provide the functionality described by the directory name. If there is no such target, do -not create an eponymous target. - -Prefer using the short name when referring to an eponymous target (`//x` instead of `//x:x`). If -you are in the same package, prefer the local reference (`:x` instead of `//x`). - -## Visibility - -Do not set the default visibility of a package to `//visibility:public`. `//visibility:public` -should be individually set for targets in the project's public API. These could be libraries which -are designed to be depended on by external projects or binaries that could be used by an external -project's build process. - -Otherwise, visibility should be scoped as tightly as possible, while still allowing access by tests -and reverse dependencies. Prefer using `__pkg__` to `__subpackages__`. - -## Dependencies - -Dependencies should be restricted to direct dependencies (dependencies needed by the sources listed -in the rule). Do not list transitive dependencies. - -Package-local dependencies should be listed first and referred to in a way compatible with the -[References to targets in the current package](#references-to-targets-in-the-current-package) -section above (not by their absolute package name). - -## Globs - -Do not use recursive globs (e.g., `glob(["**/*.java"])`). Recursive globs make BUILD files -difficult to read, as they skip subdirectories containing BUILD files. Non-recursive globs are -generally acceptable, see language-specific advice below for details. - -Indicate "no targets" with `[]`. Do not use a glob that matches nothing: it is more error-prone and -less obvious than an empty list. - -# Skylark - -## Skylark style guide - -See the [Style guide for .bzl -files](https://bazel.build/versions/master/docs/skylark/bzl-style.html) for Skylark rule guidelines. - -## Packaging rules - -See [Packaging rules](https://bazel.build/versions/master/docs/skylark/deploying.html) for advice -on how to structure and where to put new Skylark rules. - -## Rule choice - -When using a language for which Bazel has built-in rules (e.g., C++), prefer using these rules to -writing your own in Skylark. These rules are documented in the [build -encyclopedia](https://bazel.build/versions/master/docs/be/overview.html). - -# WORKSPACE files - -## Repository rules - -Prefer `http_archive` and `new_http_archive` to `git_repository`, `new_git_repository`, and -`maven_jar`. - -`git_repository` depends on jGit, which has several unpleasant bugs, and `maven_jar` uses Maven's -internal API, which generally works but is less optimized for Bazel than `http_archive`'s -downloader logic. Track the following issues filed to remediate these problems: - -- [Use `http_archive` as `git_repository`'s - backend.](https://github.com/bazelbuild/bazel/issues/2147) -- [Improve `maven_jar`'s backend.](https://github.com/bazelbuild/bazel/issues/1752) - -Do not use `bind()`. See "[Consider removing -bind](https://github.com/bazelbuild/bazel/issues/1952)" for a long discussion of its issues and -alternatives. - -## Custom BUILD files - -When using a `new_` repository rule, prefer to specify `build_file_content`, not `build_file`. - -## Skylark repository rules - -A Skylark repository rule should generally be responsible for: - -- Detecting system settings and writing them to files. -- Finding resources elsewhere on the system. -- Downloading resources from URLs. -- Generating or symlinking BUILD files into the external repository directory. - -Avoid using `repository_ctx.execute` when possible. For example, when using a non-Bazel C++ -library that has a build using Make, it is preferable to use `respository_ctx.download()` and then -write a BUILD file that builds it, instead of running `ctx.execute(["make"])`. - -# Java - -## Directory structure - -Prefer Maven's standard directory layout (sources under `src/main/java`, tests under -`src/test/java`). - -## BUILD files - -Use one BUILD file per package containing Java sources. Every BUILD file should contain one -`java_library` rule that looks like this: - -```python -java_library( - name = "directory-name", - srcs = glob(["*.java"]), - deps = [...], -) -``` - -The name of the library should be the name of the directory containing the BUILD file. The sources -should be a non-recursive glob of all Java files in the directory. - -Tests should be in a matching directory under `src/test` and depend on this library. - -# C++ - -## BUILD files - -Each BUILD file should contain one `cc_library` rule target per compilation unit in the directory. -C++ libraries should be as fine-grained as possible to provide as much incrementality as possible. - -If there is a single source file in `srcs`, the library should be named based on that C++ file's -name. This library should contain a C++ file(s), any matching header file(s), and the library's -direct dependencies. For example, - -```python -cc_library( - name = "mylib", - srcs = ["mylib.cc"], - hdrs = ["mylib.h"], - deps = [":lower-level-lib"] -) -``` - -There should be one `cc_test` rule target per `cc_library` target in the file. The `cc_test`'s -source should be a file named `[libname]_test.cc`. For example, a test for the target above might -look like: - -``` -cc_test( - name = "mylib_test", - srcs = ["mylib_test.cc"], - deps = [":mylib"] -) -``` - -## Include paths - -All include paths should be relative to the workspace directory. Use `includes` only if a public -header needs to be widely used at a non-workspace-relative path (for legacy or `third_party` code). -Otherwise, prefer to use the `copts` attribute, not the `includes` attribute. - -Using `cc_inc_library` is discouraged, prefer `copts` or `includes`. -See [the design document](https://docs.google.com/document/d/18qUWh0uUiJBv6ZOySvp6DEV0NjVnBoEy-r-ZHa9cmhU/edit#heading=h.kmep1cl5ym9k) -on C++ include directories for reasoning. - -# Protos - -## Recommended Code Organization - -- One `proto_library` rule per `.proto` file. -- A file named `foo.proto` will be in a rule named `foo_proto`, which is located in the same - package. -- A `[language]_proto_library` that wraps a `proto_library` named `foo_proto` should be called - `foo_[language]_proto`, and be located in the same package. diff --git a/site/versions/master/docs/external.md b/site/versions/master/docs/external.md deleted file mode 100644 index dae89f88bc..0000000000 --- a/site/versions/master/docs/external.md +++ /dev/null @@ -1,203 +0,0 @@ ---- -layout: documentation -title: External Dependencies ---- - -# Working with external dependencies - -External dependencies can be specified in the `WORKSPACE` file of the -[workspace directory](/docs/build-ref.html#workspace). This `WORKSPACE` file -uses the same syntax as BUILD files, but allows a different set of -rules. The full list of rules are in the Build Encyclopedia's -[Workspace Rules](/docs/be/workspace.html). - -External dependencies are all downloaded and symlinked under a directory named -`external`. You can see this directory by running: - -``` -ls $(bazel info output_base)/external -``` - -Note that running `bazel clean` will not actually delete the external -directory. To remove all external artifacts, use `bazel clean --expunge`. - -## Supported types of external dependencies - -A few basic types of external dependencies can be used: - -- [Dependencies on other Bazel projects](#bazel-projects) -- [Dependencies on non-Bazel projects](#non-bazel-projects) -- [Dependencies on external packages](#external-packages) - -<a name="bazel-projects"></a> -### Depending on other Bazel projects - -If you want to use targets from a second Bazel project, you can -use -[`local_repository`](http://bazel.build/docs/be/workspace.html#local_repository), -[`git_repository`](https://bazel.build/docs/be/workspace.html#git_repository) -or [`http_archive`](http://bazel.build/docs/be/workspace.html#http_archive) -to symlink it from the local filesystem, reference a git repository or download -it (respectively). - -For example, suppose you are working on a project, `my-project/`, and you want -to depend on targets from your coworker's project, `coworkers-project/`. Both -projects use Bazel, so you can add your coworker's project as an external -dependency and then use any targets your coworker has defined from your own -BUILD files. You would add the following to `my_project/WORKSPACE`: - -```python -local_repository( - name = "coworkers-project", - path = "/path/to/coworkers-project", -) -``` - -If your coworker has a target `//foo:bar`, your project can refer to it as -`@coworkers-project//foo:bar`. - -<a name="non-bazel-projects"></a> -### Depending on non-Bazel projects - -Rules prefixed with `new_` (e.g., -[`new_local_repository`](http://bazel.build/docs/be/workspace.html#new_local_repository), -[`new_git_repository`](https://bazel.build/docs/be/workspace.html#new_git_repository) -and [`new_http_archive`](http://bazel.build/docs/be/workspace.html#new_http_archive) -) allow you to create targets from projects that do not use Bazel. - -For example, suppose you are working on a project, `my-project/`, and you want -to depend on your coworker's project, `coworkers-project/`. Your coworker's -project uses `make` to build, but you'd like to depend on one of the .so files -it generates. To do so, add the following to `my_project/WORKSPACE`: - -```python -new_local_repository( - name = "coworkers-project", - path = "/path/to/coworkers-project", - build_file = "coworker.BUILD", -) -``` - -`build_file` specifies a BUILD file to overlay on the existing project, for -example: - -```python -java_library( - name = "some-lib", - srcs = glob(["**"]), - visibility = ["//visibility:public"], -) -``` - -You can then depend on `@coworkers-project//:some-lib` from your project's BUILD -files. - -<a name="external-packages"></a> -### Depending on external packages - -#### Maven repositories - -Use the rule [`maven_jar`](https://bazel.build/versions/master/docs/be/workspace.html#maven_jar) -(and optionally the rule [`maven_server`](https://bazel.build/versions/master/docs/be/workspace.html#maven_server)) -to download a jar from a Maven repository and make it available as a Java -dependency. - -## Fetching dependencies - -By default, external dependencies are fetched as needed during `bazel build`. If -you would like to disable this behavior or prefetch dependencies, use -[`bazel fetch`](http://bazel.build/docs/bazel-user-manual.html#fetch). - -## Using Proxies - -Bazel will pick up proxy addresses from the `HTTPS_PROXY` and `HTTP_PROXY` -environment variables and use these to download HTTP/HTTPS files (if specified). - -<a name="transitive-dependencies"></a> -## Transitive dependencies - -Bazel only reads dependencies listed in your `WORKSPACE` file. If your project -(`A`) depends on another project (`B`) which list a dependency on a third -project (`C`) in its `WORKSPACE` file, you'll have to add both `B` -and `C` to your project's `WORKSPACE` file. This requirement can balloon the -`WORKSPACE` file size, but hopefully limits the chances of having one library -include `C` at version 1.0 and another include `C` at 2.0. - -## Generate a `WORKSPACE` file - -Bazel provides a tool to help generate these expansive `WORKSPACE` files, called -`generate_workspace`. This tool is not included with the binary installer, so -you'll need to clone the [GitHub repo](https://github.com/bazelbuild/bazel) to -use it. We recommend using the tag corresponding to your current version of -bazel, which you can check by running `bazel version`. - -`cd` to the GitHub clone, `git checkout` the appropriate tag, and run the -following to build the tool and see usage: - -``` -bazel run //src/tools/generate_workspace -``` - -Note that you need run this command from your Bazel source folder even if you -build your binary from source. - -You can specify directories containing Bazel projects (i.e., directories -containing a `WORKSPACE` file), Maven projects (i.e., directories containing a -`pom.xml` file), or Maven artifact coordinates directly. For example: - -```bash -$ bazel run //src/tools/generate_workspace -- \ -> --maven_project=/path/to/my/project \ -> --bazel_project=/path/to/skunkworks \ -> --bazel_project=/path/to/teleporter/project \ -> --artifact=groupId:artifactId:version \ -> --artifact=groupId:artifactId:version -Wrote: -/tmp/1437415510621-0/2015-07-20-14-05-10.WORKSPACE -/tmp/1437415510621-0/2015-07-20-14-05-10.BUILD -``` - -The `WORKSPACE` file will contain the transitive dependencies of the given -projects and artifacts. The `BUILD` file will contain a single target, -`transitive-deps`, that contains all of the dependencies. You can copy these -files to your project and add `transitive-deps` as a dependency of your `java_` -targets in `BUILD` files. - -If you specify multiple Bazel projects, Maven projects, or artifacts, they will -all be combined into one `WORKSPACE` file (e.g., if the Bazel project depends on -junit and the Maven project also depends on junit, junit will only appear once -as a dependency in the output). - -You may wish to curate the generated `WORKSPACE` file to ensure it is using the -correct version of each dependency. If several different versions of an artifact -are requested (by different libraries that depend on it), then -`generate_workspace` chooses a version and annotates the `maven_jar` with the -other versions requested, for example: - -```python -# org.springframework:spring:2.5.6 -# javax.mail:mail:1.4 -# httpunit:httpunit:1.6 wanted version 1.0.2 -# org.springframework:spring-support:2.0.2 wanted version 1.0.2 -# org.slf4j:nlog4j:1.2.24 wanted version 1.0.2 -maven_jar( - name = "javax_activation_activation", - artifact = "javax.activation:activation:1.1", -) -``` - -The example above indicates that `org.springframework:spring:2.5.6`, -`javax.mail:mail:1.4`, `httpunit:httpunit:1.6`, -`org.springframework:spring-support:2.0.2`, and `org.slf4j:nlog4j:1.2.24` -all depend on javax.activation. However, two of these libraries wanted -version 1.1 and three of them wanted 1.0.2. The `WORKSPACE` file is using -version 1.1, but that might not be the right version to use. - -You may also want to break `transitive-deps` into smaller targets, as it is -unlikely that all of your targets depend on the transitive closure of your -maven jars. - -## Caching of external dependencies - -Bazel caches external dependencies and re-downloads or updates them when -the `WORKSPACE` file changes. diff --git a/site/versions/master/docs/getting-started.md b/site/versions/master/docs/getting-started.md deleted file mode 100644 index 66a87e40c1..0000000000 --- a/site/versions/master/docs/getting-started.md +++ /dev/null @@ -1,97 +0,0 @@ ---- -layout: documentation -title: Getting Started ---- - -# Getting Started with Bazel - -## Setup - -Use the [installation instructions](/docs/install.html) to install a copy of -Bazel on your machine. - -## Using a Workspace - -All Bazel builds take place in a [_workspace_](/docs/build-ref.html#workspaces), -a directory on your filesystem that contains source code for the software you -want to build, as well symbolic links to directories that contain the build -outputs (for example, `bazel-bin` and `bazel-out`). The location of the -workspace directory is not significant, but it must contain a file called -`WORKSPACE` in the top-level directory; an empty file is a valid workspace. -The `WORKSPACE` file can be used to reference -[external dependencies](/docs/external.html) required to build the outputs. -One workspace can be shared among multiple projects if desired. - -```bash -touch WORKSPACE -``` - -## Creating a Build File - -To know which targets can be built in your project, Bazel inspects `BUILD` -files. They are written in Bazel's build language which is syntactically -similar to Python. Usually they are just a sequence of declarations of rules. -Each rule specifies its inputs, outputs, and a way to compute the outputs from -the inputs. - -The rule probably most familiar to people who have used `Makefile`s before (as -it is the only rule available there) is the -[genrule](/docs/be/general.html#genrule), which specifies how the output can -be generated by invoking a shell command. - -``` -genrule( - name = "hello", - outs = ["hello_world.txt"], - cmd = "echo Hello World > $@", -) -``` - -The shell command may contain [Make variables](/docs/be/make-variables.html). - -Using the above `BUILD` file, you can ask Bazel to generate the target. - -``` -$ bazel build :hello -. -INFO: Found 1 target... -Target //:hello up-to-date: - bazel-genfiles/hello_world.txt -INFO: Elapsed time: 2.255s, Critical Path: 0.07s -``` - -We note two things. First, targets are normally referred to by their -[label](/docs/build-ref.html#labels), which is specified by the -[name](/docs/be/general.html#genrule.name) attribute of the rule. (Referencing -them by the output file name is also possible, but this is not the preferred -way.) Second, Bazel puts the generated files into a separate directory (the -`bazel-genfiles` directory is actually a symbolic link) so as not to pollute -your source tree. - -Rules may use the output of other rules as input, as in the following -example. Again, the generated sources are referred to by their label. - -``` -genrule( - name = "hello", - outs = ["hello_world.txt"], - cmd = "echo Hello World > $@", -) - -genrule( - name = "double", - srcs = [":hello"], - outs = ["double_hello.txt"], - cmd = "cat $< $< > $@", -) -``` - -Finally, note that, while the [genrule](/docs/be/general.html#genrule) might -seem familiar, it usually is _not_ the best rule to use. It is preferrable to -use one of the specialized [rules](/docs/be/overview.html#rules) for various -languages. - -# Next Steps - -Next, check out the tutorial on building [Java](/docs/tutorial/java.html) -or [C++](/docs/tutorial/cpp.html) programs. diff --git a/site/versions/master/docs/install.md b/site/versions/master/docs/install.md deleted file mode 100644 index fe7c46e650..0000000000 --- a/site/versions/master/docs/install.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -layout: documentation -title: Installing Bazel ---- - -# Installing Bazel - -See the instructions for installing Bazel on: - -* [Ubuntu Linux (16.04, 15.10, and 14.04)](install-ubuntu.md) -* [Mac OS X](install-os-x.md) -* [Windows (experimental)](install-windows.md) - -For other platforms, you can try to [compile from source](install-compile-source.md). - -Required Java version: - -Java JDK 8 or later is required. To address the problem of JDK 8 not being -available on some machines, Bazel's binary installer embeds a JDK by default. - -**Note:** Homebrew and Debian packages do not contain the embedded JDK. The -shell installers are the only ones with an embedded JDK. - -Extras: - -* [Bash completion](#bash) -* [zsh completion](#zsh) - -For more information on using Bazel, see -[Getting Started with Bazel](getting-started.html). - - -## <a name="jdk7"></a>Using Bazel with JDK 7 - -Bazel version _0.5.0_ does run with JDK 7. However, starting with version -_0.5.1_ Bazel must use JDK 8. - -The installers available for _0.5.0_ are: - -* `bazel-0.5.0-installer.sh`: default version with embedded JDK -* `bazel-0.5.0-without-jdk-installer.sh`: version without embedded JDK -* `bazel-0.5.0-jdk7-installer.sh`: version compatible with JDK 7, will not be - available in later releases - -## <a name="bash"></a>Getting bash completion - -Bazel comes with a bash completion script, which the installer copies into the -`bin` directory. If you ran the installer with `--user`, this will be -`$HOME/.bazel/bin`. If you ran the installer as root, this will be -`/usr/local/bazel/bin`. - -Copy the `bazel-complete.bash` script to your completion folder -(`/etc/bash_completion.d` directory under Ubuntu). If you don't have a -completion folder, you can copy it wherever suits you and insert -`source /path/to/bazel-complete.bash` in your `~/.bashrc` file (under OS X, put -it in your `~/.bash_profile` file). - -If you built Bazel from source, the bash completion target is in the `//scripts` -package: - -1. Build it with Bazel: `bazel build //scripts:bazel-complete.bash`. -2. Copy the script `bazel-bin/scripts/bazel-complete.bash` to one of the - locations described above. - -## <a name="zsh"></a>Getting zsh completion - -Bazel also comes with a zsh completion script. To install it: - -1. Add this script to a directory on your $fpath: - - ``` - fpath[1,0]=~/.zsh/completion/ - mkdir -p ~/.zsh/completion/ - cp scripts/zsh_completion/_bazel ~/.zsh/completion - ``` - - You may have to call `rm -f ~/.zcompdump; compinit` - the first time to make it work. - -2. Optionally, add the following to your .zshrc. - - ``` - # This way the completion script does not have to parse Bazel's options - # repeatedly. The directory in cache-path must be created manually. - zstyle ':completion:*' use-cache on - zstyle ':completion:*' cache-path ~/.zsh/cache - ``` diff --git a/site/versions/master/docs/mobile-install.md b/site/versions/master/docs/mobile-install.md deleted file mode 100644 index f273871d0d..0000000000 --- a/site/versions/master/docs/mobile-install.md +++ /dev/null @@ -1,220 +0,0 @@ ---- -layout: documentation -title: mobile-install ---- - -# bazel mobile-install - -<p class="lead">Fast iterative development for Android</p> - -## TL;DR - -To install small changes to an Android app very quickly, do the following: - - 1. Find the `android_binary` rule of the app you want to install. - 2. Disable Proguard by removing the `proguard_specs` attribute. - 3. Set the `multidex` attribute to `native`. - 4. Set the `dex_shards` attribute to `10`. - 5. Connect your device running ART (not Dalvik) over USB and enable USB - debugging on it. - 6. Run `bazel mobile-install :your_target`. App startup will be a little - slower than usual. - 7. Edit the code or Android resources. - 8. Run `bazel mobile-install --incremental :your_target`. - 9. Enjoy not having to wait a lot. - -Some command line options to Bazel that may be useful: - - - `--adb` tells Bazel which adb binary to use - - `--adb_arg` can be used to add extra arguments to the command line of `adb`. - One useful application of this is to select which device you want to install - to if you have multiple devices connected to your workstation: - `bazel mobile-install --adb_arg=-s --adb_arg=<SERIAL> :your_target` - - `--start_app` automatically starts the app - -When in doubt, look at the -[example](https://github.com/bazelbuild/bazel/tree/master/examples/android) -or [contact us](https://groups.google.com/forum/#!forum/bazel-discuss). - -## Introduction - -One of the most important attributes of a developer's toolchain is speed: there -is a world of difference between changing the code and seeing it run within a -second and having to wait minutes, sometimes hours, before you get any feedback -on whether your changes do what you expect them to. - -Unfortunately, the traditional Android toolchain for building an .apk entails -many monolithic, sequential steps and all of these have to be done in order to -build an Android app. At Google, waiting five minutes to build a single-line -change was not unusual on larger projects like Google Maps. - -`bazel mobile-install` makes iterative development for Android much faster by -using a combination of change pruning, work sharding, and clever manipulation of -Android internals, all without changing any of your app's code. - -## Problems with traditional app installation - -We identified the following bottlenecks of building an Android app: - -- Dexing. By default, "dx" is invoked exactly once in the build and it does not -know how to reuse work from previous builds: it dexes every method again, even -though only one method was changed. - -- Uploading data to the device. adb does not use the full bandwidth of a USB 2.0 -connection, and larger apps can take a lot of time to upload. The entire app is -uploaded, even if only small parts have changed, for example, a resource or a -single method, so this can be a major bottleneck. - -- Compilation to native code. Android L introduced ART, a new Android runtime, -which compiles apps ahead-of-time rather than compiling them just-in-time like -Dalvik. This makes apps much faster at the cost of longer installation -time. This is a good tradeoff for users because they typically install an app -once and use it many times, but results in slower development where an app is -installed many times and each version is run at most a handful of times. - -## The approach of `bazel mobile-install` - -`bazel mobile-install `makes the following improvements: - - - Sharded dexing. After building the app's Java code, Bazel shards the class - files into approximately equal-sized parts and invokes `dx` separately on - them. `dx` is not invoked on shards that did not change since the last build. - - - Incremental file transfer. Android resources, .dex files, and native - libraries are removed from the main .apk and are stored in under a separate - mobile-install directory. This makes it possible to update code and Android - resources independently without reinstalling the whole app. Thus, - transferring the files takes less time and only the .dex files that have - changed are recompiled on-device. - - - Loading parts of the app from outside the .apk. A tiny stub application is - put into the .apk that loads Android resources, Java code and native code - from the on-device mobile-install directory, then transfers control to the - actual app. This is all transparent to the app, except in a few corner cases - described below. - -### Sharded Dexing - -Sharded dexing is reasonably straightforward: once the .jar files are built, a -[tool](https://github.com/bazelbuild/bazel/blob/master/src/tools/android/java/com/google/devtools/build/android/ziputils/DexMapper.java) -shards them into separate .jar files of approximately equal size, then invokes -`dx` on those that were changed since the previous build. The logic that -determines which shards to dex is not specific to Android: it just uses the -general change pruning algorithm of Bazel. - -The first version of the sharding algorithm simply ordered the .class files -alphabetically, then cut the list up into equal-sized parts, but this proved to -be suboptimal: if a class was added or removed (even a nested or an anonymous -one), it would cause all the classes alphabetically after it to shift by one, -resulting in dexing those shards again. Thus, we settled upon sharding not -individual classes, but Java packages instead. Of course, this still results in -dexing many shards if a new package is added or removed, but that is much less -frequent than adding or removing a single class. - -The number of shards is controlled by the BUILD file (using the -`android_binary.dex_shards` attribute). In an ideal world, Bazel would -automatically determine how many shards are best, but Bazel currently must know -the set of actions (i.e. commands to be executed during the build) before -executing any of them, so it cannot determine the optimal number of shards -because it doesn't know how many Java classes there will eventually be in the -app. Generally speaking, the more shards, the faster the build and the -installation will be, but the slower app startup becomes, because the dynamic -linker has to do more work. The sweet spot is usually between 10 and 50 shards. - -### Incremental File Transfer - -After building the app, the next step is to install it, preferably with the -least effort possible. Installation consists of the following steps: - - 1. Installing the .apk (i.e. `adb install`) - 2. Uploading the .dex files, Android resources, and native libraries to the - mobile-install directory - -There is not much incrementality in the first step: the app is either installed -or not. Bazel currently relies on the user to indicate if it should do this step -through the `--incremental` command line option because it cannot determine in -all cases if it is necessary. - -In the second step, the app's files from the build are compared to an on-device -manifest file that lists which app files are on the device and their -checksums. Any new files are uploaded to the device, any files that have changed -are updated, and any files that have been removed are deleted from the -device. If the manifest is not present, it is assumed that every file needs to -be uploaded. - -Note that it is possible to fool the incremental installation algorithm by -changing a file on the device, but not its checksum in the manifest. We could -have safeguarded against this by computing the checksum of the files on the -device, but this was deemed to be not worth the increase in installation time. - -### The Stub Application - -The stub application is where the magic to load the dexes, native code and -Android resources from the on-device `mobile-install` directory happens. - -The actual loading is implemented by subclassing `BaseDexClassLoader` and is a -reasonably well-documented technique. This happens before any of the app's -classes are loaded, so that any application classes that are in the apk can be -placed in the on-device `mobile-install` directory so that they can be updated -without `adb install`. - -This needs to happen before any of the -classes of the app are loaded, so that no application class needs to be in the -.apk which would mean that changes to those classes would require a full -re-install. - -This is accomplished by replacing the `Application` class specified in -`AndroidManifest.xml` with the -[stub application](https://github.com/bazelbuild/bazel/blob/master/src/tools/android/java/com/google/devtools/build/android/incrementaldeployment/StubApplication.java). This -takes control when the app is started, and tweaks the class loader and the -resource manager appropriately at the earliest moment (its constructor) using -Java reflection on the internals of the Android framework. - -Another thing the stub application does is to copy the native libraries -installed by mobile-install to another location. This is necessary because the -dynamic linker needs the `X` bit to be set on the files, which is not possible to -do for any location accessible by a non-root `adb`. - -Once all these things are done, the stub application then instantiates the -actual `Application` class, changing all references to itself to the actual -application within the Android framework. - -## Results - -### Performance - -In general, `bazel mobile-install` results in a 4x to 10x speedup of building -and installing large apps after a small change. We computed the following -numbers for a few Google products: - -<img src="/assets/mobile-install-performance.svg"/> - -This, of course, depends on the nature of the change: recompilation after -changing a base library takes more time. - -### Limitations - -The tricks the stub application plays don't work in every case. We have -identified the following cases where it does not work as expected: - - - When `Context` is cast to the `Application` class in - `ContentProvider#onCreate()`. This method is called during application - startup before we have a chance to replace the instance of the `Application` - class, therefore, `ContentProvider` will still reference the stub application - instead of the real one. Arguably, this is not a bug since you are not - supposed to downcast `Context` like this, but this seems to happen in a few - apps at Google. - - - Resources installed by `bazel mobile-install` are only available from within - the app. If resources are accessed by other apps via - `PackageManager#getApplicationResources()`, these resources will be from the - last non-incremental install. - - - Devices that aren't running ART. While the stub application works well on - Froyo and later, Dalvik has a bug that makes it think that the app is - incorrect if its code is distributed over multiple .dex files in certain - cases, for example, when Java annotations are used in a - [specific](https://code.google.com/p/android/issues/detail?id=78144) way. As - long as your app doesn't tickle these bugs, it should work with Dalvik, too - (note, however, that support for old Android versions isn't exactly our - focus) diff --git a/site/versions/master/docs/output_directories.md b/site/versions/master/docs/output_directories.md deleted file mode 100644 index 20858c6101..0000000000 --- a/site/versions/master/docs/output_directories.md +++ /dev/null @@ -1,131 +0,0 @@ ---- -layout: documentation -title: Output Directory Layout ---- - -# Output Directory Layout - -## Requirements - -Requirements for an output directory layout: - -* Don't collide if multiple users are building on the same box. -* Support building in multiple workspaces at the same time. -* Support building for multiple target configurations in the same workspace. -* Don't collide with any other tools. -* Be easy to access. -* Be easy to clean, even selectively. -* Is unambiguous, even if the user relies on symbolic links when changing into - his/her client directory. -* All the build state per user should be underneath one directory ("I'd like to - clean all the .o files from all my clients.") - -## Documentation of the current Bazel output directory layout - -The solution that's currently implemented: - -* Bazel must be invoked from a directory containing a WORKSPACE file. It reports - an error if it is not. We call this the _workspace directory_. -* The _outputRoot_ directory is ~/.cache/bazel. (Unless `$TEST_TMPDIR` is - set, as in a test of bazel itself, in which case this directory is used - instead.) -* We stick the Bazel user's build state beneath `outputRoot/_bazel_$USER`. This - is called the _outputUserRoot_ directory. -* Beneath the `outputUserRoot` directory, we create an `installBase` directory - whose name is "install" plus the MD5 hash of the Bazel installation manifest. -* Beneath the `outputUserRoot` directory, we also create an `outputBase` - directory whose name is the MD5 hash of the path name of the workspace - directory. So, for example, if Bazel is running in the workspace directory - `/home/user/src/my-project` (or in a directory symlinked to that one), then we - create an output base directory called: - `/home/.cache/bazel/_bazel_user/7ffd56a6e4cb724ea575aba15733d113`. -* Users can use Bazel's `--output_base` startup option to override the default - output base directory. For example, - `bazel --output_base=/tmp/bazel/output build x/y:z`. -* Users can also use Bazel's `--output_user_root` startup option to override the - default install base and output base directories. For example: - `bazel --output_user_root=/tmp/bazel build x/y:z`. - -We put symlinks "bazel-<workspace-name>" and "bazel-out", as well as -"bazel-bin", "bazel-genfiles", and "bazel-includes" in the workspace directory; -these symlinks points to some directories inside a target-specific directory -inside the output directory. These symlinks are only for the user's convenience, -as Bazel itself does not use them. Also, we only do this if the workspace -directory is writable. The names of the "bazel-bin", "bazel-genfiles", and -"bazel-include" symlinks are affected by the `--symlink_prefix` option to bazel, -but "bazel-<workspace-name>" and "bazel-out" are not. - -## Bazel internals: Directory layout - -The directories are laid out as follows: - -<pre> -<workspace-name>/ <== The workspace directory - bazel-my-project => <...my-project> <== Symlink to execRoot - bazel-out => <...bin> <== Convenience symlink to outputPath - bazel-bin => <...bin> <== Convenience symlink to most recent written bin dir $(BINDIR) - bazel-genfiles => <...genfiles> <== Convenience symlink to most recent written genfiles dir $(GENDIR) - -/home/user/.cache/bazel/ <== Root for all Bazel output on a machine: outputRoot - _bazel_$USER/ <== Top level directory for a given user depends on the user name: - outputUserRoot - install/ - fba9a2c87ee9589d72889caf082f1029/ <== Hash of the Bazel install manifest: installBase - _embedded_binaries/ <== Contains binaries and scripts unpacked from the data section of - the bazel executable on first run (e.g. helper scripts and the - main Java file BazelServer_deploy.jar) - 7ffd56a6e4cb724ea575aba15733d113/ <== Hash of the client's workspace directory (e.g. - /home/some-user/src/my-project): outputBase - action_cache/ <== Action cache directory hierarchy - This contains the persistent record of the file metadata - (timestamps, and perhaps eventually also MD5 sums) used by the - FilesystemValueChecker. - action_outs/ <== Action output directory. This contains a file with the - stdout/stderr for every action from the most recent bazel run - that produced output. - command.log <== A copy of the stdout/stderr output from the most recent bazel - command. - external/ <== The directory that remote repositories are downloaded/symlinked - into. - server/ <== The Bazel server puts all server-related files (such as socket - file, logs, etc) here. - jvm.out <== The debugging output for the server. - <workspace-name>/ <== Working tree for the Bazel build & root of symlink forest: execRoot - _bin/ <== Helper tools are linked from or copied to here. - - bazel-out/ <== All actual output of the build is under here: outputPath - local_linux-fastbuild/ <== one subdirectory per unique target BuildConfiguration instance; - this is currently encoded - bin/ <== Bazel outputs binaries for target configuration here: $(BINDIR) - foo/bar/_objs/baz/ <== Object files for a cc_* rule named //foo/bar:baz - foo/bar/baz1.o <== Object files from source //foo/bar:baz1.cc - other_package/other.o <== Object files from source //other_package:other.cc - foo/bar/baz <== foo/bar/baz might be the artifact generated by a cc_binary named - //foo/bar:baz - foo/bar/baz.runfiles/ <== The runfiles symlink farm for the //foo/bar:baz executable. - MANIFEST - <workspace-name>/ - ... - genfiles/ <== Bazel puts generated source for the target configuration here: - $(GENDIR) - foo/bar.h e.g. foo/bar.h might be a headerfile generated by //foo:bargen - testlogs/ <== Bazel internal test runner puts test log files here - foo/bartest.log e.g. foo/bar.log might be an output of the //foo:bartest test with - foo/bartest.status foo/bartest.status containing exit status of the test (e.g. - PASSED or FAILED (Exit 1), etc) - include/ <== a tree with include symlinks, generated as needed. The - bazel-include symlinks point to here. This is used for - linkstamp stuff, etc. - host/ <== BuildConfiguration for build host (user's workstation), for - building prerequisite tools, that will be used in later stages - of the build (ex: Protocol Compiler) - <packages>/ <== Packages referenced in the build appear as if under a regular workspace -</pre> - -The layout of the *.runfiles directories is documented in more detail in the places pointed to by RunfilesSupport. - -## `bazel clean` - -`bazel clean` does an `rm -rf` on the `outputPath` and the `action_cache` -directory. It also removes the workspace symlinks. The `--expunge` option -will clean the entire outputBase. diff --git a/site/versions/master/docs/rule-challenges.md b/site/versions/master/docs/rule-challenges.md deleted file mode 100644 index fece635b5a..0000000000 --- a/site/versions/master/docs/rule-challenges.md +++ /dev/null @@ -1,214 +0,0 @@ ---- -layout: documentation -title: Challenges of Writing Rules. ---- - -# Challenges of Writing Rules. - -We have heard feedback from various people that they have -difficulty to write efficient Bazel rules. There is no single root cause, but -it’s due to a combination of historical circumstances and intrinsic complexity -in the problem domain. This document attempts to give a high level overview of -the specific issues that we believe to be the main contributors. - -* Assumption: Aim for Correctness, Throughput, Ease of Use & Latency -* Assumption: Large Scale Repositories -* Assumption: BUILD-like Description Language -* Intrinsic: Remote Execution and Caching are Hard -* Historic: Hard Separation between Loading, Analysis, and Execution is - Outdated, but still affects the API -* Intrinsic: Using Change Information for Correct and Fast Incremental Builds - requires Unusual Coding Patterns -* Intrinsic: Avoiding Quadratic Time and Memory Consumption is Hard - -## Assumption: Aim for Correctness, Throughput, Ease of Use & Latency - -We assume that the build system needs to be first and foremost correct with -respect to incremental builds, i.e., for a given source tree, the output of the -same build should always be the same, regardless of what the output tree looks -like. In the first approximation, this means Bazel needs to know every single -input that goes into a given build step, such that it can rerun that step if any -of the inputs change. There are limits to how correct Bazel can get, as it leaks -some information such as date / time of the build, and ignores certain types of -changes such as changes to file attributes. Sandboxing helps ensure correctness -by preventing reads to undeclared input files. Besides the intrinsic limits of -the system, there are a few known correctness issues, most of which are related -to Fileset or the C++ rules, which are both hard problems. We have long-term -efforts to fix these. - -The second goal of the build system is to have high throughput; we are -permanently pushing the boundaries of what can be done within the current -machine allocation for a remote execution service. If the remote execution -service gets overloaded, nobody can get work done. - -Ease of use comes next, i.e., of multiple correct approaches with the same (or -similar) footprint of the remote execution service, we choose the one that is -easier to use. - -For the purpose of this document, latency denotes the time it takes from -starting a build to getting the intended result, whether that is a test log from -a passing or failing test, or an error message that a BUILD file has a -typo. - -Note that these goals often overlap; latency is as much a function of throughput -of the remote execution service as is correctness relevant for ease of use. - - -## Assumption: Large Scale Repositories - -The build system needs to operate at the scale of large repositories where large -scale means that it does not fit on a single hard drive, so it is impossible to -do a full checkout on virtually all developer machines. A medium-sized build -will need to read and parse tens of thousands of BUILD files, and evaluate -hundreds of thousands of globs. While it is theoretically possible to read all -BUILD files on a single machine, we have not yet been able to do so within a -reasonable amount of time and memory. As such, it is critical that BUILD files -can be loaded and parsed independently. - - -## Assumption: BUILD-like Description Language - -For the purpose of this document, we assume a configuration language that is -roughly similar to BUILD files, i.e., declaration of library and binary rules -and their interdependencies. BUILD files can be read and parsed independently, -and we avoid even looking at source files whenever we can (except for -existence). - - -## Intrinsic: Remote Execution and Caching are Hard - -Remote execution and caching improve build times in large repositories by -roughly two orders of magnitude compared to running the build on a single -machine. However, the scale at which it needs to perform is staggering: Google's -remote execution service is designed to handle a huge number of requests per -second, and the protocol carefully avoids unnecessary roundtrips as well as -unnecessary work on the service side. - -At this time, the protocol requires that the build system knows all inputs to a -given action ahead of time; the build system then computes a unique action -fingerprint, and asks the scheduler for a cache hit. If a cache hit is found, -the scheduler replies with the digests of the output files; the files itself are -addressed by digest later on. However, this imposes restrictions on the Bazel -rules, which need to declare all input files ahead of time. - - -## Historic: Hard Separation between Loading, Analysis, and Execution is Outdated, but still affects the API - -Technically, it is sufficient for a rule to know the input and output files of -an action just before the action is sent to remote execution. However, the -original Bazel code base had a strict separation of loading packages, then -analyzing rules using a configuration (command-line flags, essentially), and -only then running any actions. This distinction is still part of the rules API -today, even though the core of Bazel no longer requires it (more details below). - -That means that the rules API requires a declarative description of the rule -interface (what attributes it has, types of attributes). There are some -exceptions where the API allows custom code to run during the loading phase to -compute implicit names of output files and implicit values of attributes. For -example, a java_library rule named ‘foo’ implicitly generates an output named -‘libfoo.jar’, which can be referenced from other rules in the build graph. - -Furthermore, the analysis of a rule cannot read any source files or inspect the -output of an action; instead, it needs to generate a partial directed bipartite -graph of build steps and output file names that is only determined from the rule -itself and its dependencies. - - -## Intrinsic: Using Change Information for Correct and Fast Incremental Builds requires Unusual Coding Patterns - -Above, we argued that in order to be correct, Bazel needs to know all the input -files that go into a build step in order to detect whether that build step is -still up-to-date. The same is true for package loading and rule analysis, and we -have designed [Skyframe] (http://www.bazel.build/docs/skyframe.html) to handle this -in general. Skyframe is a graph library and evaluation framework that takes a -goal node (such as ‘build //foo with these options’), and breaks it down into -its constituent parts, which are then evaluated and combined to yield this -result. As part of this process, Skyframe reads packages, analyzes rules, and -executes actions. - -At each node, Skyframe tracks exactly which nodes any given node used to compute -its own output, all the way from the goal node down to the input files (which -are also Skyframe nodes). Having this graph explicitly represented in memory -allows the build system to identify exactly which nodes are affected by a given -change to an input file (including creation or deletion of an input file), doing -the minimal amount of work to restore the output tree to its intended state. - -As part of this, each node performs a dependency discovery process; i.e., each -node can declare dependencies, and then use the contents of those dependencies -to declare even further dependencies. In principle, this maps well to a -thread-per-node model. However, medium-sized builds contain hundreds of -thousands of Skyframe nodes, which isn’t easily possible with current Java -technology (and for historical reasons, we’re currently tied to using Java, so -no lightweight threads and no continuations). - -Instead, Bazel uses a fixed-size thread pool. However, that means that if a node -declares a dependency that isn’t available yet, we may have to abort that -evaluation and restart it (possibly in another thread), when the dependency is -available. This, in turn, means that nodes should not do this excessively; a -node that declares N dependencies serially can potentially be restarted N times, -costing O(N^2) time. Instead, we aim for up-front bulk declaration of -dependencies, which sometimes requires reorganizing the code, or even splitting -a node into multiple nodes to limit the number of restarts. - -Note that this technology isn’t currently available in the rules API; instead, -the rules API is still defined using the legacy concepts of loading, analysis, -and execution phases. However, a fundamental restriction is that all accesses to -other nodes have to go through the framework so that it can track the -corresponding dependencies. Regardless of the language in which the build system -is implemented or in which the rules are written (they don’t have to be the -same), rule authors must not use standard libraries or patterns that bypass -Skyframe. For Java, that means avoiding java.io.File as well as any form of -reflection, and any library that does either. Libraries that support dependency -injection of these low-level interfaces still need to be setup correctly for -Skyframe. - -This strongly suggests to avoid exposing rule authors to a full language runtime -in the first place. The danger of accidental use of such APIs is just too big - -several Bazel bugs in the past were caused by rules using unsafe APIs, even -though the rules were written by the Bazel team, i.e., by the domain experts. - - -## Intrinsic: Avoiding Quadratic Time and Memory Consumption is Hard - -To make matters worse, apart from the requirements imposed by Skyframe, the -historical constraints of using Java, and the outdatedness of the rules API, -accidentally introducing quadratic time or memory consumption is a fundamental -problem in any build system based on library and binary rules. There are two -very common patterns that introduce quadratic memory consumption (and therefore -quadratic time consumption). - -1. Chains of Library Rules - -Consider the case of a chain of library rules A depends on B, depends on C, and -so on. Then, we want to compute some property over the transitive closure of -these rules, such as the Java runtime classpath, or the C++ linker command for -each library. Naively, we might take a standard list implementation; however, -this already introduces quadratic memory consumption: the first library -contains one entry on the classpath, the second two, the third three, and so -on, for a total of 1+2+3+...+N = O(N^2) entries. - -2. Binary Rules Depending on the Same Library Rules - -Consider the case where a set of binaries that depend on the same library -rules; for example, you might have a number of test rules that test the same -library code. Let’s say out of N rules, half the rules are binary rules, and -the other half library rules. Now consider that each binary makes a copy of -some property computed over the transitive closure of library rules, such as -the Java runtime classpath, or the C++ linker command line. For example, it -could expand the command line string representation of the C++ link action. N/2 -copies of N/2 elements is O(N^2) memory. - - -### Custom Collections Classes to Avoid Quadratic Complexity - -Bazel is heavily affected by both of these scenarios, so we introduced a set of -custom collection classes that effectively compress the information in memory by -avoiding the copy at each step. Almost all of these data structures have set -semantics, so we called the class NestedSet. The majority of changes to reduce -Bazel’s memory consumption over the past several years were changes to use -NestedSet instead of whatever was previously used. - -Unfortunately, usage of NestedSet does not automatically solve all the issues; -in particular, even just iterating over a NestedSet in each rule re-introduces -quadratic time consumption. NestedSet also has some helper methods to facilitate -interoperability with normal collections classes; unfortunately, accidentally -passing a NestedSet to one of these methods leads to copying behavior, and -reintroduces quadratic memory consumption. diff --git a/site/versions/master/docs/skylark/aspects.md b/site/versions/master/docs/skylark/aspects.md deleted file mode 100644 index 5dcaf1bedf..0000000000 --- a/site/versions/master/docs/skylark/aspects.md +++ /dev/null @@ -1,191 +0,0 @@ ---- -layout: documentation -title: Aspects ---- -# Aspects - -**Status: Experimental**. We may make breaking changes to the API, but we will - help you update your code. - -Aspects allow augmenting build dependency graphs with additional information -and actions. Some typical scenarios when aspects can be useful: - -* IDEs that integrate Bazel can use aspects to collect information about the - project -* Code generation tools can leverage aspects to execute on their inputs in - "target-agnostic" manner. As an example, BUILD files can specify a hierarchy - of [protobuf](https://developers.google.com/protocol-buffers/) library - definitions, and language-specific rules can use aspects to attach - actions generating protobuf support code for a particular language - -## Aspect basics - -Bazel BUILD files provide a description of a project’s source code: what source -files are part of the project, what artifacts (_targets_) should be built from -those files, what the dependencies between those files are, etc. Bazel uses -this information to perform a build, that is, it figures out the set of actions -needed to produce the artifacts (such as running compiler or linker) and -executes those actions. Bazel accomplishes this by constructing a _dependency -graph_ between targets and visiting this graph to collect those actions. - -Consider the following BUILD file: - -```python -java_library(name = 'W', ...) -java_library(name = 'Y', deps = [':W'], ...) -java_library(name = 'Z', deps = [':W'], ...) -java_library(name = 'Q', ...) -java_library(name = 'T', deps = [':Q'], ...) -java_library(name = 'X', deps = [':Y',':Z'], runtime_deps = [':T'], ...) -``` - -This BUILD file defines a dependency graph shown in Fig 1. - -<img src="build-graph.png" alt="Build Graph" width="250px" /> - -Bazel analyzes this dependency graph by calling implementations of -[rules](rules.md) (in this case "java_library" starting from leaves of -the dependency graph). These implementations generate actions that build -artifacts (such as Jar files), and provide information (such as locations -and names of those artifacts) to their dependencies in providers that -they return. Their dependencies can access those providers through the -[Target object](lib/Target.html). In other words, every target -defined in the BUILD file generates a node in the dependency graph, and -the appropriate rule implementation function is called for every node. - -Aspects are similar to rules in that they have an implementation function that -generates actions and returns providers. However, their power comes from -the way the dependency graph is built for them. An aspect has an implementation -and a list of all attributes it propagates along. Consider an aspect A that -propagates along attributes named "deps". This aspect can be applied to -a target X, yielding an aspect application node A(X). During its application, -aspect A is applied recursively to all targets that X refers to in its "deps" -attribute (all attributes in A's propagation list). Thus a single act of -applying aspect A to a target X yields a "shadow graph" of the original -dependency graph of targets (see Fig.2). - - - -The only edges that are shadowed are the edges along the attributes in -the propagation set, thus the `runtime_deps` edge is not shadowed in this -example. An aspect implementation function is then invoked on all nodes in -the shadow graph similar to how rule implementations are invoked on the nodes -of the original graph. - -## Defining aspects - -Aspect definitions are similiar to rule definitions. Let's take a look at -the example: - -```python -metal_proto_aspect = aspect(implementation = _metal_proto_aspect_impl, - attr_aspects = ["deps"], - attrs = { - "_protoc" : attr.label( - default=Label("//tools:metal_protoc"), - executable = True - ) - } -) -``` - -Just like a rule, an aspect has an implementation function. ``attr_aspects`` -specify the aspect's propagation set: a list of attributes of rules along which -the aspect propagates. - -``attrs`` defines a set of attributes for aspects. Aspects are only allowed -to have private attributes of types ``label`` or ``label_list``. Attributes -can be used to specify dependencies on tools or libraries that are needed -for actions generated by aspects. - -### Implementation functions - -Aspect implementation functions are similiar to the rule implementation -functions. They return [providers](rules.md#providers), can generate -[actions](rules.md#actions) and take two arguments: - -* `target`: the [target](lib/Target.html) the aspect is being applied to. -* `ctx`: [`ctx`](lib/ctx.html) object that can be used to access attributes and - generate outputs and actions. - -Example: - -```python -def _metal_proto_aspect_impl(target, ctx): - # For every `src` in proto_library, generate an output file - proto_sources = [f for src in ctx.rule.attr.srcs - for f in src.files] - outputs = [ctx.new_file(f.short_path + ".metal") - for f in proto_sources] - ctx.action( - executable = ctx.executable._protoc, - argument = ... - inputs = proto_sources - outputs = outputs) - transitive_outputs = depset(outputs) - for dep in ctx.rule.attr.deps: - transitive_outputs = transitive_outputs | dep.metal_proto.transitive_outputs - return struct( - metal_proto = struct(direct_outputs = outputs, - transitive_outputs = transitive_outputs)) -``` - -The implementation function can access the attributes of the target rule via -[`ctx.rule.attr`](lib/ctx.html#rule). It can examine providers that are -provided by the target to which it is applied (via the `target` argument). - -Just like a rule implementation function, an aspect implementation function -returns a struct of providers that are accessible to its dependencies. - -* The set of providers for an aspect application A(X) is the union of providers - that come from the implementation of a rule for target X and from - the implementation of aspect A. It is an error if a target and an aspect that - is applied to it each provide a provider with the same name. -* For the aspect implementation, the values of attributes along which - the aspect is propagated (from the `attr_aspect` list) are replaced with - the results of an application of the aspect to them. For example, if target - X has Y and Z in its deps, `ctx.rule.attr.deps` for A(X) will be [A(Y), A(Z)]. - In the `_metal_proto_aspect_impl` function above, ctx.rule.attr.deps will be - Target objects that are the results of applying the aspect to the 'deps' - of the original target to which the aspect has been applied. - That allows the aspect to examine `metal_proto` provider on them. - - -## Applying aspects - -Aspect propagation can be initiated either from a rule or from the command line. - -### Applying aspects to rule attributes - -Rules can specify that they want to apply aspects to their dependencies. -The aspects to be applied to a particular attribute can be specified -using the `aspects` parameter to `attr.label` or `attr.label_list` function: - -```python -metal_proto_library = rule(implementation = _impl, - attrs = { - 'proto_deps' : attr.label_list(aspects = [metal_proto_aspect]), - }, -) -``` - -If a rule specifies an aspect on its attributes, the values of that attribute -will be replaced by the result of aspect application to them (similar to -what happens during aspect propagation). Thus implementation of -`metal_proto_library` will have access to `metal_proto` providers -on the target objects representing its `proto_deps` attribute values. - -### Applying aspects from command line. - -Aspects can also be applied on the command line, using the `--aspects` flag: - - -``` -bazel build //java/com/company/example:main \ - --aspects path/to/extension.bzl%metal_proto_aspect -``` - -`--aspects` flag takes one argument, which is a specification of the aspect in -the format `<extension file path>%<aspect top-level name>`. - - diff --git a/site/versions/master/docs/skylark/build-style.md b/site/versions/master/docs/skylark/build-style.md deleted file mode 100644 index d7787593aa..0000000000 --- a/site/versions/master/docs/skylark/build-style.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -layout: documentation -title: Style guide for BUILD files ---- - -# BUILD file style guide - -In `BUILD` files, we take the same approach as in Go: We let the machine take care -of most formatting issues. -[Buildifier](https://github.com/bazelbuild/buildifier) is a tool that parses and -emits the source code in a standard style. Every `BUILD` file is therefore -formatted in the same automated way, which makes formatting a non-issue during -code reviews. It also makes it easier for tools to understand, edit, and -generate `BUILD` files. - -`BUILD` file formatting must match the output of `buildifier`. - -## Formatting example - -```python -package(default_visibility = ["//visibility:public"]) - -py_test( - name = "foo_test", - srcs = glob(["*.py"]), - data = [ - "//data/production/foo:startfoo", - "//foo", - "//third_party/java/jdk:jdk-k8", - ], - flaky = 1, - deps = [ - ":check_bar_lib", - ":foo_data_check", - ":pick_foo_port", - "//pyglib", - "//testing/pybase", - ], -) -``` - -## File structure - -We recommend to use the following order (every element is optional): - - * Package description (a comment) - - * All `load()` statements - - * The `package()` function. - - * Calls to rules and macros - -Buildifier makes a distinction between a standalone comment and a comment -attached to an element. If a comment is not attached to a specific element, use -an empty line after it. The distinction is important when doing automated -changes (e.g. to decide if we keep or remove a comment when we delete a rule). - -```python -# Standalone comment (e.g. to make a section in a file) - -# Comment for the cc_library below -cc_library(name = "cc") -``` - -## Conventions - - * Use uppercase and underscores to declare constants (e.g. `GLOBAL_CONSTANT`), - use lowercase and underscores to declare variables (e.g. `my_variable`). - - * Labels should be canonicalized. Use `//foo/bar` instead of `//foo/bar:bar`. - Use `:bar` if it is defined in the same package. *Rationale*: It makes clear - if a label is local to a package. Sorting a list of labels is messy if all - labels do not use the same conventions. - - * Labels should never be split, even if they are longer than 79 characters. - Labels should be string literals whenever possible. Rationale: It makes - find and replace easy. It also improves readability. - - * The value of the name attribute should be a literal constant string (except - in macros). *Rationale*: External tools use the name attribute to refer a - rule. They need to find rules without having to interpret code. - -## Differences with Python style guide - -Although compatibility with -[Python style guide](https://www.python.org/dev/peps/pep-0008/) is a goal, there -are a few differences: - - * No strict line length limit. Long comments and long strings are often split - to 79 columns, but it is not required. It should not be enforced in code - reviews or presubmit scripts. *Rationale*: Labels can be long and exceed this - limit. It is common for `BUILD` files to be generated or edited by tools, which - does not go well with a line length limit. - - * Implicit string concatenation is not supported. Use the `+` operator. - *Rationale*: `BUILD` files contain many string lists. It is easy to forget a - comma, which leads to a complete different result. This has created many bugs - in the past. [See also this discussion.](https://lwn.net/Articles/551438/) - - * Use spaces around the `=` sign for keywords arguments in rules. *Rationale*: - Named arguments are much more frequent than in Python and are always on a - separate line. Spaces improve readability. This convention has been around - for a long time, and we don't think it is worth modifying all existing - `BUILD` files. - - * By default, use double quotation marks for strings. *Rationale*: This is not - specified in the Python style guide, but it recommends consistency. So we - decided to use only double-quoted strings. Many languages use double-quotes - for string literals. - - * Use a single blank line between two top-level definitions. *Rationale*: The - structure of a `BUILD` file is not like a typical Python file. It has only - top-level statements. Using a single-blank line makes `BUILD` files shorter. diff --git a/site/versions/master/docs/skylark/bzl-style.md b/site/versions/master/docs/skylark/bzl-style.md deleted file mode 100644 index 66c2d2918e..0000000000 --- a/site/versions/master/docs/skylark/bzl-style.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -layout: documentation -title: Style guide for bzl files ---- - -# .bzl file style guide - -## Style - -* When in doubt, follow the - [Python style guide](https://www.python.org/dev/peps/pep-0008/). - -* Code should be documented using - [docstrings](https://www.python.org/dev/peps/pep-0257/). Use a docstring at - the top of the file, and a docstring for each public function. - -* Variables and function names use lowercase with words separated by underscores - (`[a-z][a-z0-9_]*`), e.g. `cc_library`. Top-level private values start with - one underscore. Bazel enforces that private values cannot be used from other - files. - -* As in BUILD files, there is no strict line length limit as labels can be long. - When possible, try to use at most 79 characters per line. - -* In keyword arguments, spaces around the equal sign are optional. In general, - we follow the BUILD file convention when calling macros and native rules, and - the Python convention for other functions, e.g. - -```python -def fct(name, srcs): - filtered_srcs = my_filter(source=srcs) - native.cc_library( - name = name, - srcs = filtered_srcs, - ) -``` - -## Macros - -A [macro](macros.md) is a function which instantiates one or many rules during -the loading phase. - -* Macros must accept a name attribute and each invocation should specify a name. - The generated name attribute of rules should include the name attribute as a - prefix. For example, `my_macro(name = "foo")` can generate a rule `foo` and a - rule `foo_gen`. *Rationale*: Users should be able to find easily which macro - generated a rule. Also, automated refactoring tools need a way to identify a - specific rule to edit. - -* When calling a macro, use only keyword arguments. *Rationale*: This is for - consistency with rules, it greatly improves readability. - diff --git a/site/versions/master/docs/skylark/concepts.md b/site/versions/master/docs/skylark/concepts.md deleted file mode 100644 index 8caf553dfb..0000000000 --- a/site/versions/master/docs/skylark/concepts.md +++ /dev/null @@ -1,220 +0,0 @@ ---- -layout: documentation -title: Extensions - Overview ---- -# Overview - - -## Loading an extension - -Extensions are files with the `.bzl` extension. Use the `load` statement to -import a symbol from an extension. - -```python -load("//build_tools/rules:maprule.bzl", "maprule") -``` - -This code will load the file `build_tools/rules/maprule.bzl` and add the -`maprule` symbol to the environment. This can be used to load new rules, -functions or constants (e.g. a string, a list, etc.). Multiple symbols can be -imported by using additional arguments to the call to `load`. Arguments must -be string literals (no variable) and `load` statements must appear at -top-level, i.e. they cannot be in a function body. - -`load` also supports aliases, i.e. you can assign different names to the -imported symbols. - -```python -load("//build_tools/rules:maprule.bzl", maprule_alias = "maprule") -``` - -You can define multiple aliases within one `load` statement. Moreover, the -argument list can contain both aliases and regular symbol names. The following -example is perfectly legal (please note when to use quotation marks). - -```python -load(":my_rules.bzl", "some_rule", nice_alias = "some_other_rule") -``` - -In a `.bzl` file, symbols starting with `_` are private and cannot be loaded -from another file. Visibility doesn't affect loading (yet): you don't need to -use `exports_files` to make a `.bzl` file visible. - -## Macros and rules - -A [macro](macros.md) is a function that instantiates rules. It is useful when a -BUILD file is getting too repetitive or too complex, as it allows you to reuse -some code. The function is evaluated as soon as the BUILD file is read. After -the evaluation of the BUILD file, Bazel has little information about macros: if -your macro generates a `genrule`, Bazel will behave as if you wrote the -`genrule`. As a result, `bazel query` will only list the generated `genrule`. - -A [rule](rules.md) is more powerful than a macro. It can access Bazel internals -and have full control over what is going on. It may for example pass information -to other rules. - -If you want to reuse simple logic, start with a macro. If a macro becomes -complex, it is often a good idea to make it a rule. Support for a new language -is typically done with a rule. Rules are for advanced users: we expect that most -people will never have to write one, they will only load and call existing -rules. - -## Evaluation model - -A build consists of three phases. - -* **Loading phase**. First, we load and evaluate all extensions and all BUILD - files that are needed for the build. The execution of the BUILD files simply - instantiates rules (each time a rule is called, it gets added to a graph). - This is where macros are evaluated. - -* **Analysis phase**. The code of the rules is executed (their `implementation` - function), and actions are instantiated. An action describes how to generate - a set of outputs from a set of inputs, e.g. "run gcc on hello.c and get - hello.o". It is important to note that we have to list explicitly which - files will be generated before executing the actual commands. In other words, - the analysis phase takes the graph generated by the loading phase and - generates an action graph. - -* **Execution phase**. Actions are executed, when at least one of their outputs is - required. If a file is missing or if a command fails to generate one output, - the build fails. Tests are also run during this phase. - -Bazel uses parallelism to read, parse and evaluate the `.bzl` files and `BUILD` -files. A file is read at most once per build and the result of the evaluation is -cached and reused. A file is evaluated only once all its dependencies (`load()` -statements) have been resolved. By design, loading a `.bzl` file has no visible -side-effect, it only defines values and functions. - -Bazel tries to be clever: it uses dependency analysis to know which files must -be loaded, which rules must be analyzed, and which actions must be executed. For -example, if a rule generates actions that we don't need for the current build, -they will not be executed. - -## Backward-incompatible changes - -As we make changes and polish the extension mechanism, old features may be -removed and new features that are not backwards-compatible may be added. - -Each release, new incompatible changes will be behind a flag with its default -value set to `false`. In later releases, the flag will be enabled by default, or -the flag will be removed entirely. - -To check if your code will be compatible with future releases: - -* build your code with the flag `--all_incompatible_changes`, or -* use boolean flags to enable/disable specific incompatible changes. - -This following are the planned incompatible changes that are implemented and -guarded behind flags. - -### Set constructor - -We are removing the `set` constructor. Use `depset` instead. `set` and `depset` -are equivalent, you just need to do search and replace to update the old code. - -We are doing this to reduce confusion between the specialized -[depset](depsets.md) data structure and Python's set datatype. - -* Flag: `--incompatible_disallow_set_constructor=true` -* Introduced in: 0.5.1 - -### Keyword-only arguments - -Keyword-only parameters are parameters that can be called only using their name. - -``` python -def foo(arg1, *, arg2): pass - -foo(3, arg2=3) -``` - -``` python -def bar(arg1, *rest, arg2): pass - -bar(3, arg2=3) -``` - -In both examples, `arg2` must be named at the call site. To preserve syntactic -compatibility with Python 2, we are removing this feature (which we have never -documented). - -* Flag: `--incompatible_disallow_keyword_only_args=true` -* Introduced in: 0.5.1 - -### Mutating `+=` - -We are changing `left += right` when `left` is a list. The old behavior is -equivalent to `left = left + right`, which creates a new list and assigns it to -`left`. The new behavior does not rebind `left`, but instead just mutates the -list in-place. - -``` python -def fct(): - li = [1] - alias = li - li += [2] - # Old behavior: alias == [1] - # New behavior: alias == [1, 2] -``` - -This change makes Skylark more compatible with Python and avoids performance -issues. The `+=` operator for tuples is unaffected. - -* Flag: `--incompatible_list_plus_equals_inplace=true` -* Introduced in: 0.5.1 - -### Dictionary concatenation - -We are removing the `+` operator on dictionaries. This includes the `+=` form -where the left-hand side is a dictionary. This is done to improve compatibility -with Python. A possible workaround is to use the `.update` method instead. - -* Flag: `--incompatible_disallow_dict_plus=true` -* Introduced in: 0.5.1 - -## Upcoming changes - -The following items are upcoming changes. - -* Comprehensions currently "leak" the values of their loop variables into the - surrounding scope (Python 2 semantics). This will be changed so that - comprehension variables are local (Python 3 semantics). - -* Previously dictionaries were guaranteed to use sorted order for their keys. - Going forward, there is no guarantee on order besides that it is - deterministic. As an implementation matter, some kinds of dictionaries may - continue to use sorted order while others may use insertion order. - -These changes concern the `load()` syntax in particular. - -* Currently a `load()` statement can appear anywhere in a file so long as it is - at the top-level (not in an indented block of code). In the future they will - be required to appear at the beginning of the file, i.e., before any - non-`load()` statement. - -* In BUILD files, `load()` can overwrite an existing variable with the loaded - symbol. This will be disallowed in order to improve consistency with .bzl - files. Use load aliases to avoid name clashes. - -* The .bzl file can be specified as either a path or a label. In the future only - the label form will be allowed. - -* Cross-package visibility restrictions do not yet apply to loaded .bzl files. - At some point this will change. In order to load a .bzl from another package - it will need to be exported, such as by using an `exports_files` declaration. - The exact syntax has not yet been decided. - - -## Profiling the code - -To profile your code and analyze the performance, use the `--profile` flag: - -``` -$ bazel build --nobuild --profile=/tmp/prof //path/to:target -$ bazel analyze-profile /tmp/prof --html --html_details -``` - -Then, open the generated HTML file (`/tmp/prof.html` in the example). - - diff --git a/site/versions/master/docs/skylark/cookbook.md b/site/versions/master/docs/skylark/cookbook.md deleted file mode 100644 index 28303e2457..0000000000 --- a/site/versions/master/docs/skylark/cookbook.md +++ /dev/null @@ -1,950 +0,0 @@ ---- -layout: documentation -title: Extensions examples ---- -# Extensions examples - -## <a name="macro"></a>Macro creating a rule - -An example of a macro creating a rule. - -`empty.bzl`: - -```python -def _impl(ctx): - print("This rule does nothing") - -empty = rule(implementation=_impl) -``` - -`extension.bzl`: - -```python -# Loading the rule. The rule doesn't have to be in a separate file. -load("//pkg:empty.bzl", "empty") - -def macro(name, visibility=None): - # Creating the rule. - empty(name = name, visibility = visibility) -``` - -`BUILD`: - -```python -load("//pkg:extension.bzl", "macro") - -macro(name = "myrule") -``` - -## <a name="macro_native"></a>Macro creating a native rule - -An example of a macro creating a native rule. Native rules are special rules -that are automatically available (without <code>load</code>). They are -accessed using the <a href="lib/native.html">native</a> module. - -`extension.bzl`: - -```python -def macro(name, visibility=None): - # Creating a native genrule. - native.genrule( - name = name, - outs = [name + '.txt'], - cmd = 'echo hello > $@', - visibility = visibility, - ) -``` - -`BUILD`: - -```python -load("//pkg:extension.bzl", "macro") - -macro(name = "myrule") -``` - -## <a name="macro_compound"></a>Macro multiple rules - -There's currently no easy way to create a rule that directly uses the -action of a native rule. You can work around this using macros: - -```python -def _impl(ctx): - return struct([...], - # When instrumenting this rule, again hide implementation from - # users. - instrumented_files( - source_attributes = ["srcs", "csrcs"], - dependency_attributes = ["deps", "cdeps"])) - -# This rule is private and can only be accessed from the current file. -_cc_and_something_else_binary = rule(implementation=_impl) - - -# This macro is public, it's the public interface to instantiate the rule. -def cc_and_something_else_binary(name, srcs, deps, csrcs, cdeps): - cc_binary_name = "%s.cc_binary" % name - - native.cc_binary( - name = cc_binary_name, - srcs = csrcs, - deps = cdeps, - visibility = ["//visibility:private"] - ) - - _cc_and_something_else_binary( - name = name, - srcs = srcs, - deps = deps, - # A label attribute so that this depends on the internal rule. - cc_binary = cc_binary_name, - # Redundant labels attributes so that the rule with this target name knows - # about everything it would know about if cc_and_something_else_binary - # were an actual rule instead of a macro. - csrcs = csrcs, - cdeps = cdeps) -``` - - -## <a name="conditional-instantiation"></a>Conditional instantiation - -Macros can look at previously instantiated rules. This is done with -`native.existing_rule`, which returns information on a single rule defined in the same -`BUILD` file, eg., - -```python -native.existing_rule("descriptor_proto") -``` - -This is useful to avoid instantiating the same rule twice, which is an -error. For example, the following macro will simulate a test suite, -instantiating tests for diverse flavors of the same test. - -`extension.bzl`: - -```python -def system_test(name, test_file, flavor): - n = "system_test_%s_%s_test" % (test_file, flavor) - if native.existing_rule(n) == None: - native.py_test( - name = n, - srcs = [ - "test_driver.py", - test_file, - ], - args = ["--flavor=" + flavor], - ) - return n - -def system_test_suite(name, flavors=["default"], test_files=[]): - ts = [] - for flavor in flavors: - for test in test_files: - ts.append(system_test(name, test, flavor)) - native.test_suite(name = name, tests = ts) -``` - -In the following BUILD file, note how `(basic_test.py, fast)` is emitted for -both the `smoke` test suite and the `thorough` test suite. - -`BUILD`: - -```python -load("//pkg:extension.bzl", "system_test_suite") - -# Run all files through the 'fast' flavor. -system_test_suite( - name = "smoke", - flavors = ["fast"], - test_files = glob(["*_test.py"]), -) - -# Run the basic test through all flavors. -system_test_suite( - name = "thorough", - flavors = [ - "fast", - "debug", - "opt", - ], - test_files = ["basic_test.py"], -) -``` - - -## <a name="aggregation"></a>Aggregating over the BUILD file - -Macros can collect information from the BUILD file as processed so far. We call -this aggregation. The typical example is collecting data from all rules of a -certain kind. This is done by calling -<a href="lib/native.html#existing_rules">native.existing\_rules</a>, which -returns a dictionary representing all rules defined so far in the current BUILD -file. The dictionary has entries of the form `name` => `rule`, with the values -using the same format as `native.existing_rule`. - -```python -def archive_cc_src_files(tag): - """Create an archive of all C++ sources that have the given tag.""" - all_src = [] - for r in native.existing_rules().values(): - if tag in r["tags"] and r["kind"] == "cc_library": - all_src.append(r["srcs"]) - native.genrule(cmd = "zip $@ $^", srcs = all_src, outs = ["out.zip"]) -``` - -Since `native.existing_rules` constructs a potentially large dictionary, you should avoid -calling it repeatedly within BUILD file. - -## <a name="empty"></a>Empty rule - -Minimalist example of a rule that does nothing. If you build it, the target will -succeed (with no generated file). - -`empty.bzl`: - -```python -def _impl(ctx): - # You may use print for debugging. - print("This rule does nothing") - -empty = rule(implementation=_impl) -``` - -`BUILD`: - -```python -load("//pkg:empty.bzl", "empty") - -empty(name = "nothing") -``` - -## <a name="attr"></a>Rule with attributes - -Example of a rule that shows how to declare attributes and access them. - -`printer.bzl`: - -```python -def _impl(ctx): - # You may use print for debugging. - print("Rule name = %s, package = %s" % (ctx.label.name, ctx.label.package)) - - # This prints the labels of the deps attribute. - print("There are %d deps" % len(ctx.attr.deps)) - for i in ctx.attr.deps: - print("- %s" % i.label) - # A label can represent any number of files (possibly 0). - print(" files = %s" % [f.path for f in i.files]) - -printer = rule( - implementation=_impl, - attrs={ - # Do not declare "name": It is added automatically. - "number": attr.int(default = 1), - "deps": attr.label_list(allow_files=True), - }) -``` - -`BUILD`: - -```python -load("//pkg:printer.bzl", "printer") - -printer( - name = "nothing", - deps = [ - "BUILD", - ":other", - ], -) - -printer(name = "other") -``` - -If you execute this file, some information is printed as a warning by the -rule. No file is generated. - -## <a name="shell"></a>Simple shell command - -Example of a rule that runs a shell command on an input file specified by -the user. The output has the same name as the rule, with a `.size` suffix. - -While convenient, Shell commands should be used carefully. Generating the -command-line can lead to escaping and injection issues. It can also create -portability problems. It is often better to declare a binary target in a -BUILD file and execute it. See the example [executing a binary](#execute-bin). - -`size.bzl`: - -```python -def _impl(ctx): - output = ctx.outputs.out - input = ctx.file.file - # The command may only access files declared in inputs. - ctx.action( - inputs=[input], - outputs=[output], - progress_message="Getting size of %s" % input.short_path, - command="stat -L -c%%s %s > %s" % (input.path, output.path)) - -size = rule( - implementation=_impl, - attrs={"file": attr.label(mandatory=True, allow_files=True, single_file=True)}, - outputs={"out": "%{name}.size"}, -) -``` - -`foo.txt`: - -``` -Hello -``` - -`BUILD`: - -```python -load("//pkg:size.bzl", "size") - -size( - name = "foo_size", - file = "foo.txt", -) -``` - -## <a name="file"></a>Write string to a file - -Example of a rule that writes a string to a file. - -`file.bzl`: - -```python -def _impl(ctx): - output = ctx.outputs.out - ctx.file_action(output=output, content=ctx.attr.content) - -file = rule( - implementation=_impl, - attrs={"content": attr.string()}, - outputs={"out": "%{name}.txt"}, -) -``` - -`BUILD`: - -```python -load("//pkg:file.bzl", "file") - -file( - name = "hello", - content = "Hello world", -) -``` - - -## <a name="execute-bin"></a>Execute a binary - -This rule executes an existing binary. In this particular example, the -binary is a tool that merges files. During the analysis phase, we cannot -access any arbitrary label: the dependency must have been previously -declared. To do so, the rule needs a label attribute. In this example, we -will give the label a default value and make it private (so that it is not -visible to end users). Keeping the label private can simplify maintenance, -since you can easily change the arguments and flags you pass to the tool. - -`execute.bzl`: - -```python -def _impl(ctx): - # The list of arguments we pass to the script. - args = [ctx.outputs.out.path] + [f.path for f in ctx.files.srcs] - # Action to call the script. - ctx.action( - inputs=ctx.files.srcs, - outputs=[ctx.outputs.out], - arguments=args, - progress_message="Merging into %s" % ctx.outputs.out.short_path, - executable=ctx.executable._merge_tool) - -concat = rule( - implementation=_impl, - attrs={ - "srcs": attr.label_list(allow_files=True), - "out": attr.output(mandatory=True), - "_merge_tool": attr.label(executable=True, cfg="host", allow_files=True, - default=Label("//pkg:merge")) - } -) -``` - -Any executable target can be used. In this example, we will use a -`sh_binary` rule that concatenates all the inputs. - -`BUILD`: - -``` -load("execute", "concat") - -concat( - name = "sh", - srcs = [ - "header.html", - "body.html", - "footer.html", - ], - out = "page.html", -) - -# This target is used by the shell rule. -sh_binary( - name = "merge", - srcs = ["merge.sh"], -) -``` - -`merge.sh`: - -```python -#!/bin/bash - -out=$1 -shift -cat $* > $out -``` - -`header.html`: - -``` -<html><body> -``` - -`body.html`: - -``` -content -``` - -`footer.html`: - -``` -</body></html> -``` - -## <a name="execute"></a>Execute an input binary - -This rule has a mandatory `binary` attribute. It is a label that can refer -only to executable rules or files. - -`execute.bzl`: - -```python -def _impl(ctx): - # ctx.new_file is used for temporary files. - # If it should be visible for user, declare it in rule.outputs instead. - f = ctx.new_file(ctx.configuration.bin_dir, "hello") - # As with outputs, each time you declare a file, - # you need an action to generate it. - ctx.file_action(output=f, content=ctx.attr.input_content) - - ctx.action( - inputs=[f], - outputs=[ctx.outputs.out], - executable=ctx.executable.binary, - progress_message="Executing %s" % ctx.executable.binary.short_path, - arguments=[ - f.path, - ctx.outputs.out.path, # Access the output file using - # ctx.outputs.<attribute name> - ] - ) - -execute = rule( - implementation=_impl, - attrs={ - "binary": attr.label(cfg="host", mandatory=True, allow_files=True, - executable=True), - "input_content": attr.string(), - "out": attr.output(mandatory=True), - }, -) -``` - -`a.sh`: - -```bash -#!/bin/bash - -tr 'a-z' 'A-Z' < $1 > $2 -``` - -`BUILD`: - -```python -load("//pkg:execute.bzl", "execute") - -execute( - name = "e", - input_content = "some text", - binary = "a.sh", - out = "foo", -) -``` - -## <a name="runfiles"></a>Runfiles and location substitution - -`execute.bzl`: - -```python -def _impl(ctx): - executable = ctx.outputs.executable - command = ctx.attr.command - # Expand the label in the command string to a runfiles-relative path. - # The second arg is the list of labels that may be expanded. - command = ctx.expand_location(command, ctx.attr.data) - # Create the output executable file with command as its content. - ctx.file_action( - output=executable, - content=command, - executable=True) - - return [DefaultInfo( - # Create runfiles from the files specified in the data attribute. - # The shell executable - the output of this rule - can use them at runtime. - # It is also possible to define data_runfiles and default_runfiles. - # However if runfiles is specified it's not possible to define the above - # ones since runfiles sets them both. - # Remember, that the struct returned by the implementation function needs - # to have a field named "runfiles" in order to create the actual runfiles - # symlink tree. - runfiles=ctx.runfiles(files=ctx.files.data) - )] - -execute = rule( - implementation=_impl, - executable=True, - attrs={ - "command": attr.string(), - "data": attr.label_list(cfg="data", allow_files=True), - }, -) -``` - -`data.txt`: - -``` -Hello World! -``` - -`BUILD`: - -```python -load("//pkg:execute.bzl", "execute") - -execute( - name = "e", - # The location will be expanded to "pkg/data.txt", and it will reference - # the data.txt file in runfiles when this target is invoked as - # "bazel run //pkg:e". - command = "cat $(location :data.txt)", - data = [':data.txt'] -) -``` - -## <a name="late-bound"></a>Computed dependencies - -Bazel needs to know about all dependencies before doing the analysis phase and -calling the implementation function. Dependencies can be computed based on the -rule attributes: to do so, use a function as the default -value of an attribute (the attribute must be private and have type `label` or -`list of labels`). The parameters of this function must correspond to the -attributes that are accessed in the function body. - -Note: For legacy reasons, the function takes the configuration as an additional -parameter. Please do not rely on the configuration since it will be removed in -the future. - -The example below computes the md5 sum of a file. The file can be preprocessed -using a filter. The exact dependencies depend on the filter chosen by the user. - -`hash.bzl`: - -```python -_filters = { - "comments": Label("//pkg:comments"), - "spaces": Label("//pkg:spaces"), - "none": None, -} - -def _get_filter(filter, cfg=None): # requires attribute "filter" - # Return the value for the attribute "_filter_bin" - # It can be a label or None. - return _filters[filter] - -def _impl(ctx): - src = ctx.file.src - - if not ctx.attr._filter_bin: - # Skip the processing - processed = src - else: - processed = ctx.new_file(ctx.label.name + "_processed") - # Run the selected binary - ctx.action( - outputs = [processed], - inputs = [ctx.file.src], - progress_message="Apply filter '%s'" % ctx.attr.filter, - arguments = [ctx.file.src.path, processed.path], - executable = ctx.executable._filter_bin) - - # Compute the hash - out = ctx.outputs.text - ctx.action( - outputs = [out], - inputs = [processed], - command = "md5sum < %s > %s" % (processed.path, out.path)) - -md5_sum = rule( - implementation=_impl, - attrs={ - "filter": attr.string(values=_filters.keys(), default="none"), - "src": attr.label(mandatory=True, single_file=True, allow_files=True), - "_filter_bin": attr.label(default=_get_filter, executable=True), - }, - outputs = {"text": "%{name}.txt"}) -``` - -`BUILD`: - -```python -load("//pkg:hash.bzl", "md5_sum") - -md5_sum( - name = "hash", - src = "hello.txt", - filter = "spaces", -) - -sh_binary( - name = "comments", - srcs = ["comments.sh"], -) - -sh_binary( - name = "spaces", - srcs = ["spaces.sh"], -) -``` - -`hello.txt`: - -``` -Hello World! -``` - -`comments.sh`: - -``` -#!/bin/bash -grep -v '^ *#' $1 > $2 # Remove lines with only a Python-style comment -``` - -`spaces.sh`: - -``` -#!/bin/bash -tr -d ' ' < $1 > $2 # Remove spaces -``` - -## <a name="mandatory-providers"></a>Mandatory providers - -In this example, rules have a `number` attribute. Each rule adds its -number with the numbers of its transitive dependencies, and write the -result in a file. This shows how to transfer information from a dependency -to its dependents. - -`sum.bzl`: - -```python -NumberInfo = provider() - -def _impl(ctx): - result = ctx.attr.number - for dep in ctx.attr.deps: - result += dep[NumberInfo].number - ctx.file_action(output=ctx.outputs.out, content=str(result)) - - # Return the provider with result, visible to other rules. - return [NumberInfo(number=result)] - -sum = rule( - implementation=_impl, - attrs={ - "number": attr.int(default=1), - # All deps must provide all listed providers. - "deps": attr.label_list(providers=[NumberInfo]), - }, - outputs = {"out": "%{name}.sum"} -) -``` - -`BUILD`: - -```python -load("//pkg:sum.bzl", "sum") - -sum( - name = "n", - deps = ["n2", "n5"], -) - -sum( - name = "n2", - number = 2, -) - -sum( - name = "n5", - number = 5, -) -``` - -## <a name="optional-providers"></a>Optional providers - -This is a similar example, but dependencies may not provide a number. - -`sum.bzl`: - -```python -NumberInfo = provider() - -def _impl(ctx): - result = ctx.attr.number - for dep in ctx.attr.deps: - if NumberInfo in dep: - result += dep[NumberInfo].number - ctx.file_action(output=ctx.outputs.out, content=str(result)) - - # Return the provider with result, visible to other rules. - return [NumberInfo(number=result)] - -sum = rule( - implementation=_impl, - attrs={ - "number": attr.int(default=1), - "deps": attr.label_list(), - }, - outputs = {"out": "%{name}.sum"} -) -``` - -`BUILD`: - -```python -load("//pkg:sum.bzl", "sum") - -sum( - name = "n", - deps = ["n2", "n5"], -) - -sum( - name = "n2", - number = 2, -) - -sum( - name = "n5", - number = 5, -) -``` - -## <a name="outputs-executable"></a>Default executable output - -This example shows how to create a default executable output. - -`extension.bzl`: - -```python -def _impl(ctx): - ctx.file_action( - # Access the executable output file using ctx.outputs.executable. - output=ctx.outputs.executable, - content="#!/bin/bash\necho Hello!", - executable=True - ) - # The executable output is added automatically to this target. - -executable_rule = rule( - implementation=_impl, - executable=True -) -``` - -`BUILD`: - -```python -load("//pkg:extension.bzl", "executable_rule") - -executable_rule(name = "my_rule") -``` - -## <a name="outputs-default"></a>Default outputs - -This example shows how to create default outputs for a rule. - -`extension.bzl`: - -```python -def _impl(ctx): - ctx.file_action( - # Access the default outputs using ctx.outputs.<output name>. - output=ctx.outputs.my_output, - content="Hello World!" - ) - # The default outputs are added automatically to this target. - -rule_with_outputs = rule( - implementation=_impl, - outputs = { - # %{name} is substituted with the rule's name - "my_output": "%{name}.txt" - } -) -``` - -`BUILD`: - -```python -load("//pkg:extension.bzl", "rule_with_outputs") - -rule_with_outputs(name = "my_rule") -``` - -## <a name="outputs-custom"></a>Custom outputs - -This example shows how to create custom (user defined) outputs for a rule. -This rule takes a list of output file name templates from the user and -creates each of them containing a "Hello World!" message. - -`extension.bzl`: - -```python -def _impl(ctx): - # Access the custom outputs using ctx.outputs.<attribute name>. - for output in ctx.outputs.outs: - ctx.file_action( - output=output, - content="Hello World!" - ) - # The custom outputs are added automatically to this target. - -rule_with_outputs = rule( - implementation=_impl, - attrs={ - "outs": attr.output_list() - } -) -``` - -`BUILD`: - -```python -load("//pkg:extension.bzl", "rule_with_outputs") - -rule_with_outputs( - name = "my_rule", - outs = ["my_output.txt"] -) -``` - -## <a name="master-rule"></a>Master rules - -This example shows how to create master rules to bind other rules together. The -code below uses genrules for simplicity, but this technique is more useful with -other rules. For example, if you need to compile C++ files, you can reuse -`cc_library`. - -`extension.bzl`: - -```python -def _impl(ctx): - # Aggregate the output files from the depending rules - files = depset() - files += ctx.attr.dep_rule_1.files - files += ctx.attr.dep_rule_2.files - return [DefaultInfo(files=files)] - -# This rule binds the depending rules together -master_rule = rule( - implementation=_impl, - attrs={ - "dep_rule_1": attr.label(), - "dep_rule_2": attr.label() - } -) - -def macro(name, cmd, input): - # Create the depending rules - name_1 = name + "_dep_1" - name_2 = name + "_dep_2" - native.genrule( - name = name_1, - cmd = cmd, - outs = [name_1 + ".txt"] - ) - native.genrule( - name = name_2, - cmd = "echo " + input + " >$@", - outs = [name_2 + ".txt"] - ) - # Create the master rule - master_rule( - name = name, - dep_rule_1 = ":" + name_1, - dep_rule_2 = ":" + name_2 - ) -``` - -`BUILD`: - -```python -load("//pkg:extension.bzl", "macro") - -# This creates the target :my_rule -macro( - name = "my_rule", - cmd = "echo something > $@", - input = "Hello World" -) -``` - -## <a name="debugging-tips"></a>Debugging tips - -Here are some examples on how to debug macros and rules using -<a href="lib/globals.html#print">print</a>. - -`debug.bzl`: - -```python -print("print something when the module is loaded") - -def _impl(ctx): - print("print something when the rule implementation is executed") - print(type("abc")) # prints string, the type of "abc" - print(dir(ctx)) # prints all the fields and methods of ctx - print(dir(ctx.attr)) # prints all the attributes of the rule - # prints the objects each separated with new line - print("object1", "object2", sep="\n") - -debug = rule(implementation=_impl) -``` - -`BUILD`: - -```python -load("//pkg:debug.bzl", "debug") - -debug( - name = "printing_rule" -) -``` - diff --git a/site/versions/master/docs/skylark/deploying.md b/site/versions/master/docs/skylark/deploying.md deleted file mode 100644 index 658445175e..0000000000 --- a/site/versions/master/docs/skylark/deploying.md +++ /dev/null @@ -1,144 +0,0 @@ ---- -layout: documentation -title: Deploying new Skylark rules ---- -# Deploying new Skylark rules - -This documentation is for Skylark rule writers who are planning to make their -rules available to others. - -## Where to put new rules - -In general, new rules should go into their own GitHub repository under your -organization. Contact the [bazel-dev mailing -list](https://groups.google.com/forum/#!forum/bazel-dev) if you feel like your -rules belong in the bazelbuild organization. - -You can see lots of examples of what your repository should look like on GitHub: -see all of the repositories named `rules_whatever`. In particular, -[rules_scala](https://github.com/bazelbuild/rules_scala) is a nice example of -how to set up your repo. - -Rules can be grouped either by language (e.g., Scala) or some notion of platform -(e.g., Android). - -## What a rule repository should contain - -Every rule repository should have a certain layout so that users can quickly -understand new rules. - -For example, suppose we are writing new Skylark rules for the (make-believe) -chaiscript language. We would have the following structure: - -``` -.travis.yml -README.md -WORKSPACE -chaiscript/ - BUILD - chaiscript.bzl -tests/ - BUILD - some_test.sh - another_test.py -examples/ - BUILD - bin.chai - lib.chai - test.chai -``` - -### README.md - -At the top level, there should be a README.md that contains (at least) what -users will need to copy-paste into their WORKSPACE file to use your rule. -In general, this will be a `git_repository` pointing to your GitHub repo and -a macro call that downloads/configures any tools your rule needs. For example, -for the [Go -rules](https://github.com/bazelbuild/rules_go/blob/master/README.md#setup), this -looks like: - -``` -git_repository( - name = "io_bazel_rules_go", - remote = "https://github.com/bazelbuild/rules_go.git", - tag = "0.0.2", -) -load("@io_bazel_rules_go//go:def.bzl", "go_repositories") - -go_repositories() -``` - -If your rules depend on another repository's rules, specify both in the -`README.md` (see the [Skydoc rules](https://github.com/bazelbuild/skydoc#setup), -which depend on the Sass rules, for an example of this). - -### Tests - -There should be tests that verify that the rules are working as expected. This -can either be in the standard location for the language the rules are for or a -`tests/` directory at the top level. - -### Optional: Examples - -It is useful to users to have an `examples/` directory that shows users a couple -of basic ways that the rules can be used. - -## Testing - -Set up Travis as described in their [getting started -docs](https://docs.travis-ci.com/user/getting-started/). Then add a -`.travis.yml` file to your repository with the following content: - -``` -language: - - java -jdk: - - oraclejdk8 # Building Bazel requires JDK8. -before_install: - - wget https://github.com/bazelbuild/bazel/archive/0.3.0.zip # Replace with desired version - - unzip 0.3.0.zip - - cd bazel-0.3.0 - - ./compile.sh - - sudo cp output/bazel /usr/bin/bazel - - cd .. - - rm -rf bazel-0.3.0 -script: - - bazel build //... - - bazel test //... -``` - -Right now Bazel has to be compiled from source, as Travis does not support a -version of GCC that works with the precompiled Bazel binaries. Thus, the -`before_install` steps download the Bazel source, compile it, and "install" the -Bazel binary in /usr/bin. - -If your repository is under the [bazelbuild organization](https://github.com/bazelbuild), -contact the [bazel-dev](https://groups.google.com/forum/#!forum/bazel-dev) list -to have it added to [ci.bazel.build](http://ci.bazel.build). - -## Documentation - -See the [Skydoc documentation](https://github.com/bazelbuild/skydoc) for -instructions on how to comment your rules so that documentation can be generated -automatically. - -## FAQs - -### Why can't we add our rule to the Bazel GitHub repository? - -We want to decouple rules from Bazel releases as much as possible. It's clearer -who owns individual rules, reducing the load on Bazel developers. For our users, -decoupling makes it easier to modify, upgrade, downgrade, and replace rules. -Contributing to rules can be lighter weight than contributing to Bazel - -depending on the rules -, including full submit access to the corresponding -GitHub repository. Getting submit access to Bazel itself is a much more involved -process. - -The downside is a more complicated one-time installation process for our users: -they have to copy-paste a rule into their WORKSPACE file, as shown in the -README section above. - -We used to have all of the Skylark rules in the Bazel repository (under -`//tools/build_rules` or `//tools/build_defs`). We still have a couple rules -there, but we are working on moving the remaining rules out. diff --git a/site/versions/master/docs/skylark/index.md b/site/versions/master/docs/skylark/index.md deleted file mode 100644 index dc336c99da..0000000000 --- a/site/versions/master/docs/skylark/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -layout: documentation -title: Extensions ---- - -# Extensions -Skylark is the name of the extension mechanism in Bazel. It lets you add support -for new languages and tools by writing [custom build rules](rules.md). You can -also compose existing rules into [macros](macros.md). - -## Getting started - -Read the [concepts](concepts.md) behind Skylark and try the -[cookbook examples](cookbook.md). To go further, read about the -[standard library](lib/globals.html). ---> - diff --git a/site/versions/master/docs/skylark/macros.md b/site/versions/master/docs/skylark/macros.md deleted file mode 100644 index 7782198287..0000000000 --- a/site/versions/master/docs/skylark/macros.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -layout: documentation -title: Macros ---- -# Macros - -## Macro creation - -A macro is a function called from the BUILD file that can instantiate rules. -Macros don't give additional power, they are just used for encapsulation and -code reuse. By the end of the [loading phase](concepts.md#evaluation-model), -macros don't exist anymore, and Bazel sees only the set of rules they created. - -Native rules (i.e. rules that don't need a `load()` statement) can be -instantiated from the [native](lib/native.html) module, e.g. - -```python -def my_macro(name, visibility=None): - native.cc_library( - name = name, - srcs = ["main.cc"], - visibility = visibility, - ) -``` - -If you need to know the package name (i.e. which BUILD file is calling the -macro), use the constant [PACKAGE_NAME](lib/globals.html#PACKAGE_NAME). - -## Examples - -* [Macro creating rules](cookbook.md#macro). - -* [Macro creating native rules](cookbook.md#macro_native). - -* [Macro combining multiple rules](cookbook.md#macro_compound). - -## Debugging - -* `bazel query --output=build //my/path:all` will show you how the BUILD file - looks after evaluation. All macros, globs, loops are expanded. Known - limitation: `select` expressions are currently not shown in the output. - -* You may filter the output based on `generator_function` (which function - generated the rules) or `generator_name` (the name attribute of the macro), - e.g. - ```bash - $ bazel query --output=build 'attr(generator_function, my_macro, //my/path:all)' - ``` - -* To find out where exactly the rule `foo` is generated in a BUILD file, you - can try the following trick. Insert this line near the top of the BUILD - file: `cc_library(name = "foo")`. Run Bazel. You will get an exception when - the rule `foo` is created (due to a name conflict), which will show you the - full stack trace. - -* You can also use [print](lib/globals.html#print) for debugging. It displays - the message as a warning during the loading phase. Except in rare cases, - either remove `print` calls, or make them conditional under a `debugging` - parameter that defaults to `False` before submitting the code to the depot. - -## Errors - -If you want to throw an error, use the [fail](lib/globals.html#fail) function. -Explain clearly to the user what went wrong and how to fix their BUILD file. It -is not possible to catch an error. - -``` -def my_macro(name, deps, visibility=None): - if len(deps) < 2: - fail("Expected at least two values in deps") - # ... -``` - -## Conventions - -* All public functions (functions that don't start with underscore) that - instantiate rules must have a `name` argument. This argument should not be - optional (don't give a default value). - -* Public functions should use a docstring following [Python - conventions](https://www.python.org/dev/peps/pep-0257/#one-line-docstrings). - -* In BUILD files, the `name` argument of the macros must be a keyword argument - (not a positional argument). - -* The `name` attribute of rules generated by a macro should include the name - argument as a prefix. For example, `macro(name = "foo")` can generate a - `cc_library` `foo` and a genrule `foo_gen`. - -* In most cases, optional parameters should have a default value of `None`. - `None` can be passed directly to native rules, which treat it the same as if - you had not passed in any argument. Thus, there is no need to replace it - with `0`, `False`, or `[]` for this purpose. Instead, the macro should defer - to the rules it creates, as their defaults may be complex or may change over - time. Additionally, a parameter that is explicitly set to its default value - looks different than one that is never set (or set to `None`) when accessed - through the query language or build-system internals. - -* Macros should have an optional `visibility` argument. - -## Full example - -The typical use-case for a macro is when you want to reuse a genrule, e.g. - -``` -genrule( - name = "file", - outs = ["file.txt"], - cmd = "$(location generator) some_arg > $@", - tools = [":generator"], -) -``` - -If you want to generate another file with different arguments, you may want to -extract this code to a function. - -The BUILD file will become simply: - -``` -load("//path:generator.bzl", "file_generator") - -file_generator( - name = "file", - arg = "some_arg", -) -``` - -In order to keep BUILD files clean and declarative, you must put the function in -a separate `.bzl` file. For example, write the definition of the macro in -`path/generator.bzl`: - -``` -def file_generator(name, arg, visibility=None): - native.genrule( - name = name, - outs = [name + ".txt"], - cmd = "$(location generator) %s > $@" % arg, - tools = ["//test:generator"], - visibility = visibility, - ) -``` - -When you want to investigate what a macro does, use the following command to -see the expanded form: - -``` -$ bazel query --output=build :file -# /absolute/path/test/ext.bzl:42:3 -genrule( - name = "file", - tools = ["//test:generator"], - outs = ["//test:file.txt"], - cmd = "$(location generator) some_arg > $@", -) -``` - diff --git a/site/versions/master/docs/skylark/repository_rules.md b/site/versions/master/docs/skylark/repository_rules.md deleted file mode 100644 index c6890062ef..0000000000 --- a/site/versions/master/docs/skylark/repository_rules.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -layout: documentation -title: Repository Rules ---- -# Repository Rules - -**Status: Experimental**. We may make breaking changes to the API, but we will - announce them and help you update your code. - -An [external repository](/docs/external.md) is a rule that can be used only -in the `WORKSPACE` file and enable non-hermetic operation at the loading phase -of Bazel. Each external repository rule creates its own workspace, with its -own BUILD files and artifacts. They can be used to depend on third-party -libraries (such as Maven packaged libraries) but also to generate BUILD files -specific to the host Bazel is running on. - -## Repository Rule creation - -In a `.bzl` file, use the -[repository_rule](lib/globals.html#repository_rule) function to create a new -repository rule and store it in a global variable. - -A custom repository rule can be used just like a native repository rule. It -has a mandatory `name` attribute and every target present in its build files -can be referred as `@<name>//package:target` where `<name>` is the value of the -`name` attribute. - -The rule is loaded when you explicitly build it, or if it is a dependency of -the build. In this case, Bazel will execute its `implementation` function. This -function describe how to creates the repository, its content and BUILD files. - -## Attributes - -An attribute is a rule argument, such as `url` or `sha256`. You must list -the attributes and their types when you define a repository rule. - -```python -local_repository = repository_rule( - implementation=_impl, - local=True, - attrs={"path": attr.string(mandatory=True)}) -``` - -`name` attributes are implicitly defined for all `repository_rule`s. -To access an attribute, use `repository_ctx.attr.<attribute_name>`. -The name of a repository rule is accessible with `repository_ctx.name`. - -If an attribute name starts with `_` it is private and users cannot set it. - -## Implementation function - -Every repository rule requires an `implementation` function. It contains the -actual logic of the rule and is executed strictly in the Loading Phase. -The function has exactly one input parameter, `repository_ctx`, and should -always returns `None`. The input parameter `repository_ctx` can be used to -access attribute values, and non-hermetic functions (finding a binary, -executing a binary, creating a file in the repository or downloading a file -from the Internet). See [the library](lib/repository_ctx.html) for more -context. Example: - -```python -def _impl(repository_ctx): - repository_ctx.symlink(repository_ctx.attr.path, "") - -local_repository = repository_rule( - implementation=_impl, - ...) -``` - -## When is the implementation function executed? - -If the repository is declared as `local` then change in a dependency -in the dependency graph (including the WORKSPACE file itself) will -cause an execution of the implementation function. - -The implementation function can be _restarted_ if a dependency it -request is _missing_. The beginning of the implementation function -will be re-executed after the dependency has been resolved. - -File given as a label are declared as dependencies, so requesting it -might interrupt the function and restart it later, re-executing the -part up till there. - -Finally, for non-`local` repositories, only a change in the following -dependencies might cause a restart: - -- Skylark files needed to define the repository rule. -- Declaration of the repository rule in the `WORKSPACE` file. -- Value of any environment variable declared with the `environ` -attribute of the -[`repository_rule`](https://bazel.build/versions/master/docs/skylark/lib/globals.html#repository_rule) -function. The value of those environment variable can be enforced from -the command line with the -[`--action_env`](/docs/command-line-reference.html#flag--action_env) -flag (but this flag will invalidate every action of the build). -- Content of any file used and referred to by a label (e.g., - `//mypkg:label.txt` not `mypkg/label.txt`). - -## Examples - -- [C++ auto-configured toolchain](https://github.com/bazelbuild/bazel/blob/ac29b78000afdb95afc7e97efd2b1299ebea4dac/tools/cpp/cc_configure.bzl#L288): -it uses a repository rule to automatically create the -C++ configuration files for Bazel by looking for the local C++ compiler, the -environment and the flags the C++ compiler supports. -- - [Go repositories](https://github.com/bazelbuild/rules_go/blob/67bc217b6210a0922d76d252472b87e9a6118fdf/go/private/go_repositories.bzl#L195) - uses several `repository_rule` to defines the list of dependencies - needed to use the Go rules. -- - [maven_jar](https://github.com/bazelbuild/bazel/a110ac400190c90a45856f15482c8d0952c542f5/master/tools/build_defs/repo/maven_rules.bzl#L276) - is a reimplementation of the native `maven_jar` rule using the - `maven` tool. - diff --git a/site/versions/master/docs/skylark/rules.md b/site/versions/master/docs/skylark/rules.md deleted file mode 100644 index 47f27abb13..0000000000 --- a/site/versions/master/docs/skylark/rules.md +++ /dev/null @@ -1,596 +0,0 @@ ---- -layout: documentation -title: Rules ---- -# Rules - -**Status: Experimental**. We may make breaking changes to the API, but we will - help you update your code. - -A rule defines a series of [actions](#actions) that Bazel should perform on -inputs to get a set of outputs. For example, a C++ binary rule might take a set -of `.cpp` files (the inputs), run `g++` on them (the action), and return an -executable file (the output). - -Note that, from Bazel's perspective, `g++` and the standard C++ libraries are -also inputs to this rule. As a rule writer, you must consider not only the -user-provided inputs to a rule, but also all of the tools and libraries required -to execute the actions (called _implicit inputs_). - -Before creating or modifying any rule, make sure you are familiar with the -[extensibility model](concepts.md) (understand the three phases and the -differences between macros and rules). - -## Rule creation - -In a `.bzl` file, use the [rule](lib/globals.html#rule) -function to create a new rule and store it in a global variable: - -```python -my_rule = rule(...) -``` - -See [the cookbook](cookbook.md#empty) for examples. The rule can then be -loaded by BUILD files: - -```python -load('//some/pkg:whatever.bzl', 'my_rule') -``` - -A custom rule can be used just like a native rule. It has a mandatory `name` -attribute, you can refer to it with a label, and you can see it in -`bazel query`. - -The rule is analyzed when you explicitly build it, or if it is a dependency of -the build. In this case, Bazel will execute its `implementation` function. This -function decides what the outputs of the rule are and how to build them (using -[actions](#actions)). During the [analysis phase](concepts.md#evaluation-model), -no external command can be executed. Instead, actions are registered and -will be run in the execution phase, if their output is needed for the build. - -## Attributes - -An attribute is a rule argument, such as `srcs` or `deps`. You must list -the attributes and their types when you define a rule. - -```python -sum = rule( - implementation = _impl, - attrs = { - "number": attr.int(default = 1), - "deps": attr.label_list(), - }, -) -``` - -The following attributes are implicitly added to every rule: `deprecation`, -`features`, `name`, `tags`, `testonly`, `visibility`. Test rules also have the -following attributes: `args`, `flaky`, `local`, `shard_count`, `size`, -`timeout`. - -Labels listed in `attr` will be inputs to the rule. - -To access an attribute in a rule's implementation, use -`ctx.attr.<attribute_name>`. The name and the package of a rule are available -with `ctx.label.name` and `ctx.label.package`. - -See [an example](cookbook.md#attr) of using `attr` in a rule. - -### <a name="private-attributes"></a> Private Attributes - -In Python, we use one leading underscore(`_`) for non-public methods and -instance variables (see [PEP-8][1]). - -Similarly, if an attribute name starts with `_` it is private and users cannot -set it. -It is useful in particular for label attributes (your rule will have an -implicit dependency on this label). - -```python -metal_compile = rule( - implementation = _impl, - attrs = { - "srcs": attr.label_list(), - "_compiler": attr.label( - default = Label("//tools:metalc"), - allow_single_file = True, - executable = True, - ), - }, -) -``` - -## Implementation function - -Every rule requires an `implementation` function. It contains the actual logic -of the rule and is executed strictly in the -[analysis phase](concepts.md#evaluation-model). The function has exactly one -input parameter, `ctx`, and it may return the [runfiles](#runfiles) -and [providers](#providers) of the rule. The input parameter `ctx` can be used -to access attribute values, outputs and dependent targets, and files. It also -has some helper functions. See [the library](lib/ctx.html) for more context. -Example: - -```python -def _impl(ctx): - ... - return [DefaultInfo(runfiles=...), MyInfo(...)] - -my_rule = rule( - implementation = _impl, - ... -) -``` - -## Files - -There are two kinds of files: files stored in the file system and generated -files. For each generated file, there must be one and only one generating -action, and each action must generate one or more output files. Bazel will throw -an error otherwise. - -## Targets - -Every build rule corresponds to exactly one target. A target can create -[actions](#actions), can have dependencies (which can be files or -other build rules), [output files](#output-files) (generated by -its actions), and [providers](#providers). - -A target `y` depends on target `x` if `y` has a label or label list type -attribute where `x` is declared: - -```python -my_rule( - name = "x", -) - -my_rule( - name = "y", - deps = [":x"], -) -``` - -In the above case, it's possible to access targets declared in `my_rule.deps`: - -```python -def _impl(ctx): - for dep in ctx.attr.deps: - # Do something with dep - ... - -my_rule = rule( - implementation = _impl, - attrs = { - "deps": attr.label_list(), - }, - ... -) -``` - -## <a name="output-files"></a> Output files - -A target can declare output files, which must be generated by the target's -actions. There are three ways to create output files: - -* If the rule is marked `executable`, it creates an output file of the same name - as the rule's. [See example](cookbook.md#outputs-executable) - -* The rule can declare default `outputs`, which are always generated. - [See example](cookbook.md#outputs-default) - -* The rule can have output or output list type attributes. In that case the - output files come from the actual attribute values. - [See example](cookbook.md#outputs-custom) - -Each output file must have exactly one generating action. See the -[library](lib/ctx.html#outputs) for more context. - -## Default outputs - -Every rule has a set of default outputs. This is used: - -* When the user runs `bazel build` on your target. Bazel will build the default - outputs of the rule. - -* When the target is used as a dependency of another rule. A rule can access - the default outputs by using [target.files](lib/Target.html#files). - This is the case, for example, if you use a rule in the `srcs` attribute of a - `genrule`. - -Use the `files` provider to specify the default outputs of a rule. -If left unspecified, it will contain all the declared outputs. - -```python -def _impl(ctx): - # ... - return DefaultInfo(files=depset([file1, file2])) -``` - -This can be useful for exposing files generated with -[ctx.new_file](lib/ctx.html#new_file). You can also have "implicit -outputs", i.e., files that are declared in the rule, but not in the default -outputs (like `_deploy.jar` in `java_binary`). - -## Actions - -An action describes how to generate a set of outputs from a set of inputs, for -example "run gcc on hello.c and get hello.o". When an action is created, Bazel -doesn't run the command immediately. It registers it in a graph of dependencies, -because an action can depend on the output of another action (e.g. in C, -the linker must be called after compilation). In the execution phase, Bazel -decides which actions must be run and in which order. - -There are three ways to create actions: - -* [ctx.action](lib/ctx.html#action), to run a command. -* [ctx.file_action](lib/ctx.html#file_action), to write a string to a file. -* [ctx.template_action](lib/ctx.html#template_action), to generate a file from a template. - -Actions take a set (which can be empty) of input files and generate a (non-empty) -set of output files. -The set of input and output files must be known during the -[analysis phase](concepts.md#evaluation-model). It might depend on the value -of attributes and information from dependencies, but it cannot depend on the -result of the execution. For example, if your action runs the unzip command, you -must specify which files you expect to be inflated (before running unzip). - -Actions are comparable to pure functions: They should depend only on the -provided inputs, and avoid accessing computer information, username, clock, -network, or I/O devices (except for reading inputs and writing outputs). This is -important because the output will be cached and reused. - -**If an action generates a file that is not listed in its outputs**: This is -fine, but the file will be ignored and cannot be used by other rules. - -**If an action does not generate a file that is listed in its outputs**: This is -an execution error and the build will fail. This happens for instance when a -compilation fails. - -**If an action generates an unknown number of outputs and you want to keep them -all**, you must group them in a single file (e.g., a zip, tar, or other -archive format). This way, you will be able to deterministically declare your -outputs. - -**If an action does not list a file it uses as an input**, the action execution -will most likely result in an error. The file is not guaranteed to be available -to the action, so if it **is** there, it's due to coincidence or error. - -**If an action lists a file as an input, but does not use it**: This is fine. -However, it can affect action execution order, resulting in sub-optimal -performance. - -Dependencies are resolved by Bazel, which will decide which actions are -executed. It is an error if there is a cycle in the dependency graph. Creating -an action does not guarantee that it will be executed: It depends on whether -its outputs are needed for the build. - -## Configurations - -Imagine that you want to build a C++ binary and target a different architecture. -The build can be complex and involve multiple steps. Some of the intermediate -binaries, like the compilers and code generators, have to run on your machine -(the host); some of the binaries such the final output must be built for the -target architecture. - -For this reason, Bazel has a concept of "configurations" and transitions. The -topmost targets (the ones requested on the command line) are built in the -"target" configuration, while tools that should run locally on the host are -built in the "host" configuration. Rules may generate different actions based on -the configuration, for instance to change the cpu architecture that is passed to -the compiler. In some cases, the same library may be needed for different -configurations. If this happens, it will be analyzed and potentially built -multiple times. - -By default, Bazel builds the dependencies of a target in the same configuration -as the target itself, i.e. without transitioning. When a target depends on a -tool, the label attribute will specify a transition to the host configuration. -This causes the tool and all of its dependencies to be built for the host -machine, assuming those dependencies do not themselves have transitions. - -For each [label attribute](lib/attr.html#label), you can decide whether the -dependency should be built in the same configuration, or transition to the host -configuration (using `cfg`). If a label attribute has the flag -`executable=True`, the configuration must be set explicitly. -[See example](cookbook.html#execute-a-binary) - -In general, sources, dependent libraries, and executables that will be needed at -runtime can use the same configuration. - -Tools that are executed as part of the build (e.g., compilers, code generators) -should be built for the host configuration. In this case, specify `cfg="host"` -in the attribute. - -Otherwise, executables that are used at runtime (e.g. as part of a test) should -be built for the target configuration. In this case, specify `cfg="target"` in -the attribute. - -The configuration `"data"` is present for legacy reasons and should be used for -the `data` attributes. - -## <a name="fragments"></a> Configuration Fragments - -Rules may access [configuration fragments](lib/skylark-configuration-fragment.html) -such as `cpp`, `java` and `jvm`. However, all required fragments must be -declared in order to avoid access errors: - -```python -def _impl(ctx): - # Using ctx.fragments.cpp would lead to an error since it was not declared. - x = ctx.fragments.java - ... - -my_rule = rule( - implementation = _impl, - fragments = ["java"], # Required fragments of the target configuration - host_fragments = ["java"], # Required fragments of the host configuration - ... -) -``` - -`ctx.fragments` only provides configuration fragments for the target -configuration. If you want to access fragments for the host configuration, -use `ctx.host_fragments` instead. - -## Providers - -Providers are pieces of information that a rule exposes to other rules that -depend on it. This data can include output files, libraries, parameters to pass -on a tool's command line, or anything else the depending rule should know about. -Providers are the only mechanism to exchange data between rules, and can be -thought of as part of a rule's public interface (loosely analogous to a -function's return value). - -A rule can only see the providers of its direct dependencies. If there is a rule -`top` that depends on `middle`, and `middle` depends on `bottom`, then we say -that `middle` is a direct dependency of `top`, while `bottom` is a transitive -dependency of `top`. In this case, `top` can see the providers of `middle`. The -only way for `top` to see any information from `bottom` is if `middle` -re-exports this information in its own providers; this is how transitive -information can be accumulated from all dependencies. In such cases, consider -using [depsets](depsets.md) to hold the data more efficiently without excessive -copying. - -Providers can be declared using the [provider()](lib/globals.html#provider) function: - -```python -TransitiveDataInfo = provider() -``` - -Rule implementation function can then construct and return provider instances: - -```python -def rule_implementation(ctx): - ... - return [TransitiveDataInfo(value = ["a", "b", "c"])] -``` - -`TransitiveDataInfo` acts both as a constructor for provider instances and as a key to access them. -A [target](lib/Target.html) serves as a map from each provider that the target supports, to the -target's corresponding instance of that provider. -A rule can access the providers of its dependencies using the square bracket notation (`[]`): - -```python -def dependent_rule_implementation(ctx): - ... - s = depset() - for dep_target in ctx.attr.deps: - s += dep_target[TransitiveDataInfo].value - ... -``` - -All targets have a [`DefaultInfo`](lib/globals.html#DefaultInfo) provider that can be used to access -some information relevant to all targets. - -Providers are only available during the analysis phase. Examples of usage: - -* [mandatory providers](cookbook.md#mandatory-providers) -* [optional providers](cookbook.md#optional-providers) - -> *Note:* -> Historically, Bazel also supported provider instances that are identified by strings and -> accessed as fields on the `target` object instead of as keys. This style is deprecated -> but still supported. Return legacy providers as follows: -> -```python -def rule_implementation(ctx): - ... - modern_provider = TransitiveDataInfo(value = ["a", "b", "c"]) - # Legacy style. - return struct(legacy_provider = struct(...), - another_legacy_provider = struct(...), - # The `providers` field contains provider instances that can be accessed - # the "modern" way. - providers = [modern_provider]) -``` -> To access legacy providers, use the dot notation. -> Note that the same target can define both modern and legacy providers: -> -```python -def dependent_rule_implementation(ctx): - ... - s = depset() - for dep_target in ctx.attr.deps: - x = dep_target.legacy_provider # legacy style - s += dep_target[TransitiveDataInfo].value # modern style - ... -``` -> **We recommend using modern providers for all future code.** - -## Runfiles - -Runfiles are a set of files used by the (often executable) output of a rule -during runtime (as opposed to build time, i.e. when the binary itself is -generated). -During the [execution phase](concepts.md#evaluation-model), Bazel creates a -directory tree containing symlinks pointing to the runfiles. This stages the -environment for the binary so it can access the runfiles during runtime. - -Runfiles can be added manually during rule creation and/or collected -transitively from the rule's dependencies: - -```python -def _rule_implementation(ctx): - ... - transitive_runfiles = depset() - for dep in ctx.attr.special_dependencies: - transitive_runfiles += dep.transitive_runtime_files - - runfiles = ctx.runfiles( - # Add some files manually. - files = [ctx.file.some_data_file], - # Add transitive files from dependencies manually. - transitive_files = transitive_runfiles, - # Collect runfiles from the common locations: transitively from srcs, - # deps and data attributes. - collect_default = True, - ) - # Add a field named "runfiles" to the DefaultInfo provider in order to actually - # create the symlink tree. - return [DefaultInfo(runfiles=runfiles)] -``` - -Note that non-executable rule outputs can also have runfiles. For example, a -library might need some external files during runtime, and every dependent -binary should know about them. - -Also note that if an action uses an executable, the executable's runfiles can -be used when the action executes. - -Normally, the relative path of a file in the runfiles tree is the same as the -relative path of that file in the source tree or generated output tree. If these -need to be different for some reason, you can specify the `root_symlinks` or -`symlinks` arguments. The `root_symlinks` is a dictionary mapping paths to -files, where the paths are relative to the root of the runfiles directory. The -`symlinks` dictionary is the same, but paths are implicitly prefixed with the -name of the workspace. - -```python - ... - runfiles = ctx.runfiles( - root_symlinks = {"some/path/here.foo": ctx.file.some_data_file2} - symlinks = {"some/path/here.bar": ctx.file.some_data_file3} - ) - # Creates something like: - # sometarget.runfiles/ - # some/ - # path/ - # here.foo -> some_data_file2 - # <workspace_name>/ - # some/ - # path/ - # here.bar -> some_data_file3 -``` - -If `symlinks` or `root_symlinks` is used, be careful not to map two different -files to the same path in the runfiles tree. This will cause the build to fail -with an error describing the conflict. To fix, you will need to modify your -`ctx.runfiles` arguments to remove the collision. This checking will be done for -any targets using your rule, as well as targets of any kind that depend on those -targets. - -## Output groups - -By default Bazel builds a target's -[default outputs](#default-outputs). However, a rule can also create - other outputs that are not part of a typical build but might still be useful, - such as debug information files. The facility for this is _output groups_. - -A rule can declare that a certain file belongs to a certain output group by returning -the [OutputGroupInfo](lib/globals.html#OutputGroupInfo) provider. Fields of -that provider are output group names: - -```python -def _impl(ctx): - name = ... - binary = ctx.new_file(name) - debug_file = ctx.new_file(name + ".pdb") - # ... add actions to generate these files - return [DefaultInfo(files = depset([binary])), - OutputGroupInfo(debug_files = depset([debug_file]), - all_files = depset([binary, debug_file]))] -``` - -By default, only the `binary` file will be built. -The user can specify an [`--output_groups=debug_files`](../command-line-reference.html#build) -flag on the command line. In that case, only `debug_file` will be built. If the user -specifies `--output_groups=all_files`, both `binary` and `debug_file` will be build. - -> Note: [OutputGroupInfo](skylark/lib/globals.html#OutputGroupInfo) is a regular -> [provider](#providers), and dependencies of a target can examine it using -> the `target[OutputGroupInfo]` syntax. - -## Code coverage instrumentation - -A rule can use the `instrumented_files` provider to provide information about -which files should be measured when code coverage data collection is enabled: - -```python -def _rule_implementation(ctx): - ... - return struct(instrumented_files = struct( - # Optional: File extensions used to filter files from source_attributes. - # If not provided, then all files from source_attributes will be - # added to instrumented files, if an empty list is provided, then - # no files from source attributes will be added. - extensions = ["ext1", "ext2"], - # Optional: Attributes that contain source files for this rule. - source_attributes = ["srcs"], - # Optional: Attributes for dependencies that could include instrumented - # files. - dependency_attributes = ["data", "deps"])) -``` - -[ctx.config.coverage_enabled](lib/configuration.html#coverage_enabled) notes -whether coverage data collection is enabled for the current run in general -(but says nothing about which files specifically should be instrumented). -If a rule implementation needs to add coverage instrumentation at -compile-time, it can determine if its sources should be instrumented with -[ctx.coverage_instrumented](lib/ctx.html#coverage_instrumented): - -```python -# Are this rule's sources instrumented? -if ctx.coverage_instrumented(): - # Do something to turn on coverage for this compile action -``` - -Note that function will always return false if `ctx.config.coverage_enabled` is -false, so you don't need to check both. - -If the rule directly includes sources from its dependencies before compilation -(e.g. header files), it may also need to turn on compile-time instrumentation -if the dependencies' sources should be instrumented. In this case, it may -also be worth checking `ctx.config.coverage_enabled` so you can avoid looping -over dependencies unnecessarily: - -```python -# Are this rule's sources or any of the sources for its direct dependencies -# in deps instrumented? -if ctx.config.coverage_enabled: - if (ctx.coverage_instrumented() or - any(ctx.coverage_instrumented(dep) for dep in ctx.attr.deps): - # Do something to turn on coverage for this compile action -``` - -## Executable rules - -An executable rule is a rule that users can run using `bazel run`. - -To make a rule executable, set `executable=True` in the -[rule function](lib/globals.html#rule). The `implementation` function of the -rule must generate the output file `ctx.outputs.executable`. -[See example](cookbook.md#outputs-executable) - -## Test rules - -Test rules are run using `bazel test`. - -To create a test rule, set `test=True` in the -[rule function](lib/globals.html#rule). The name of the rule must -also end with `_test`. Test rules are implicitly executable, which means that -the `implementation` function of the rule must generate the output file -`ctx.outputs.executable`. - -Test rules inherit the following attributes: `args`, `flaky`, `local`, -`shard_count`, `size`, `timeout`. - -[1]: https://www.python.org/dev/peps/pep-0008/#id46 diff --git a/site/versions/master/docs/support.md b/site/versions/master/docs/support.md deleted file mode 100644 index 407cec8e0c..0000000000 --- a/site/versions/master/docs/support.md +++ /dev/null @@ -1,257 +0,0 @@ ---- -layout: documentation -title: Get Support ---- - -# Get Support - -* Ask questions on [Stack Overflow](http://stackoverflow.com/questions/tagged/bazel) using the -`bazel` tag. -* Discuss on the [User mailing list](https://groups.google.com/forum/#!forum/bazel-discuss). -* Report bugs or feature requests in our [GitHub issue tracker](https://github.com/bazelbuild/bazel/issues). -* Find other Bazel contributors on [IRC](http://webchat.freenode.net) -(irc.freenode.net#bazel). - -# Support Policy - -We generally avoid making backwards-incompatible changes. We have several years of experience with -supporting a huge code base that is concurrently worked on by thousands of engineers every day, -and have successfully made significant changes to the core as well as to the rules without missing -a beat. We run hundreds of thousands of tests at Google before every single release to ensure that -it stays that way. - -That said, we occasionally have to make incompatible changes in order to fix bugs, to make further -improvements to the system, such as improving performance or usability, or to lock down APIs that -are known to be brittle. - -This document gives an overview of features that are widely used and that we consider stable. By -stable, we mean that the changes we make will be backwards compatible, or that we will provide a -migration path. - -It also covers features that are unstable. Either they are not yet widely used, or we are already -planning to change them significantly, possibly in ways that are not backwards compatible. - -We cannot cover everything that might change, but you can reasonably expect that we provide -advance notice on the mailing list before a major change happens. We're also happy to provide more -details, just ask on [bazel-discuss](https://groups.google.com/forum/#!forum/bazel-discuss). - -All undocumented features (attributes, rules, "Make" variables, and flags) are subject to change -at any time without prior notice. Features that are documented but marked *experimental* are also -subject to change at any time without prior notice. - -Bazel's extension language Skylark (anything you write in a `.bzl` file) is still subject to change. -We are in the process of migrating Google to Skylark, and expect the language part to extend macros -to stabilize as part of that process. Adding rules with skylark is still somewhat experimental. - -Help keep us honest: report bugs and regressions in the -[GitHub bugtracker](https://github.com/bazelbuild/bazel/issues). We will make an effort to triage all -reported issues within 2 business days. - -## Releases - -We regularly publish [binary releases of Bazel](https://github.com/bazelbuild/bazel/releases). To -that end, we announce release candidates on -[bazel-discuss](https://groups.google.com/forum/#!forum/bazel-discuss); these are binaries that have -passed all of our unit tests. Over the next few days, we regression test all applicable build -targets at Google. If you have a critical project using Bazel, we recommend that you establish an -automated testing process that tracks the current release candidate, and report any regressions. - -If no regressions are discovered, we officially release the candidate after a week. However, -regressions can delay the release of a release candidate. If regressions are found, we apply -corresponding cherry-picks to the release candidate to fix those regressions. If no further -regressions are found for two business days, but not before a week has elapsed since the first -release candidate, we release it. - -### Release versioning - -Version 0.1 is our first release marking the start of our beta phase. Until version 1.0.0, we -increase the MINOR version every time we reach a [new milestone](http://bazel.build/roadmap.html). - -Version 1.0.0 marks the end of our beta phase; afterwards, we will label each release with a -version number of the form MAJOR.MINOR.PATCH according to the -[semantic version 2.0.0 document](http://semver.org). - -## Current Status - -### Built-In Rules and the Internal API For Rules ### -We are planning a number of changes to the APIs between the core of Bazel and the built-in rules, -in order to make it easier for us to develop openly. This has the added benefit of also making it -easier for users to maintain their own rules (if written in Java instead of Skylark), if they don't -want to or cannot check this code into our repository. However, it also means our internal API is -not stable yet. In the long term, we want to move to Skylark wholesale, so we encourage contributors -to use Skylark instead of Java when writing new rules. Rewriting all of our built-in rules is going -to be a lengthy process however. - -1. We will fix the friction points that we know about, as well as those that we discover every time - we make changes that span both the internal and external depots. -2. We will drive a number of pending API cleanups to completion, as well as run anticipated cleanups - to the APIs, such as disallowing access to the file system from rule implementations (because - it's not hermetic). -3. We will enumerate the internal rule APIs, and make sure that they are appropriately marked (for - example with annotations) and documented. Just collecting a list will likely give us good - suggestions for further improvements, as well as opportunities for a more principled API review - process. -4. We will automatically check rule implementations against an API whitelist, with the intention - that API changes are implicitly flagged during code review. -5. We will work on removing (legacy) Google-internal features to reduce the amount of differences - between the internal and external rule sets. -6. We will encourage our engineers to make changes in the external depot first, and migrate them to - to the internal one afterwards. -7. We will move more of our rule implementations into the open source repository (Android, Go, - Python, the remaining C++ rules), even if we don't consider the code to be *ready* or if they are - still missing tools to work properly. -8. In order to be able to accept external contributions, our highest priority item for Skylark is a - testing framework. We encourage to write new rules in Skylark rather than in Java. - - -### Stable -We expect the following rules and features to be stable. They are widely used within Google, so -our internal testing should ensure that there are no major breakages. - -<table class="table table-condensed table-striped table-bordered"> - <colgroup> - <col class="support-col-rules" /> - <col class="support-col-notes" /> - </colgroup> - <thead> - <tr> - <th>Rules</th> - <th>Notes</th> - </tr> - </thead> - <tbody> - <tr> - <td>C/C++ rules except <code>cc_toolchain</code> and <code>cc_toolchain_suite</code></td> - <td></td> - </tr> - <tr> - <td>Java rules except <code>java_toolchain</code></td> - <td></td> - </tr> - <tr> - <td>Android rules except <code>android_ndk_repository</code> and - <code>android_sdk_repository</code></td> - <td></td> - </tr> - <tr> - <td><code>genrule</code></td> - <td></td> - </tr> - <tr> - <td><code>genquery</code></td> - <td></td> - </tr> - <tr> - <td><code>test_suite</code></td> - <td></td> - </tr> - <tr> - <td><code>filegroup</code></td> - <td></td> - </tr> - <tr> - <td><code>config_setting</code></td> - <td> - <ul> - <li>This rule is used in <code>select()</code> expressions. We have hundreds of uses, so - we expect the basic functionality to be stable. That said, there are some common use - cases that are not covered yet, or that require workarounds. For example, it's not - easily possible to select on information specified in a CROSSTOOL file, such as the - target abi. Another example is that it's not possible to OR multiple conditions, - leading to duplicated entries in the select. - </li> - </ul> - </td> - </tr> - </tbody> -</table> - - -### Unstable -These rules and features have known limitations that we will likely address in future releases. - -<table class="table table-condensed table-striped table-bordered"> - <colgroup> - <col class="support-col-rules" /> - <col class="support-col-notes" /> - </colgroup> - <thead> - <tr> - <th>Feature</th> - <th>Notes</th> - </tr> - </thead> - <tbody> - <tr> - <td><code>cc_toolchain</code> and <code>cc_toolchain_suite</code></td> - <td> - <ul> - <li>We intend to make significant changes to the way C/C++ toolchains are defined; we will - keep our published C/C++ toolchain definition(s) up to date, but we make no guarantees for - custom ones.</li> - </ul> - </td> - </tr> - <tr> - <td>iOS/Objective C rules</td> - <td> - <ul> - <li>We cannot vouch for changes made by Apple ® to the underlying tools and - infrastructure.</li> - <li>The rules are fairly new and still subject to change; we try to avoid breaking changes, - but this may not always be possible.</li> - <li>No testing support yet.</li> - </ul> - </td> - </tr> - <tr> - <td>Python rules</td> - <td> - <ul> - <li>The rules support neither Python 3, C/C++ extensions, nor packaging. - </li> - </ul> - </td> - </tr> - <tr> - <td>Extra actions (<code>extra_action</code>, <code>action_listener</code>)</td> - <td> - <ul> - <li>Extra actions expose information about Bazel that we consider to be implementation - details, such as the exact interface between Bazel and the tools we provide; as such, - users will need to keep up with changes to tools to avoid breakage.</li> - </ul> - </td> - </tr> - <tr> - <td><code>environment_group</code></td> - <td> - <ul> - <li>We're planning to use it more extensively, replacing several machine-enforceable - constraint mechanism, but there's only a handful of uses so far. We fully expect it to - work, but there's a small chance that we have to go back to the drawing board.</li> - </ul> - </td> - </tr> - <tr> - <td><code>android_ndk_repository</code> and <code>android_sdk_repository</code></td> - <td> - <ul> - <li>We don't support pre-release NDKs or SDKs at this time. Furthermore, we may still - make backwards-incompatible changes to the attributes or the semantics.</li> - </ul> - </td> - </tr> - <tr> - <td><code>Fileset</code></td> - <td> - <ul> - <li>There are vestiges of Fileset / FilesetEntry in the source code, but we do not intend to - support them in Bazel, ever.</li> - <li>They're still widely used internally, and are therefore unlikely to go away in the near - future.</li> - </ul> - </td> - </tbody> -</table> - diff --git a/site/versions/master/docs/tutorial/android-app.md b/site/versions/master/docs/tutorial/android-app.md deleted file mode 100644 index f31736b9f9..0000000000 --- a/site/versions/master/docs/tutorial/android-app.md +++ /dev/null @@ -1,295 +0,0 @@ ---- -layout: documentation -title: Tutorial - Build an Android App ---- - -# Tutorial - Build an Android App - -The sample Android app in this tutorial is a very simple application that makes -an HTTP connection to the [backend server](backend-server.md) and displays the -resulting response. - -Here, you'll do the following: - -* Review the source files for the app -* Update the `WORKSPACE` file -* Create a `BUILD` file -* Run the build -* Find the build outputs -* Run the app - -## Review the source files - -Let's take a look at the source files for the app. These are located in -`$WORKSPACE/android/`. - -The key files and directories are: - -<table class="table table-condensed table-striped"> -<thead> -<tr> -<td>Name</td> -<td>Location</td> -</tr> -</thead> -<tbody> -<tr> -<td>Manifest file</td> -<td><code>src/main/java/com/google/bazel/example/android/AndroidManifest.xml</code></td> -</tr> -<tr> -<td>Activity source file</td> -<td><code>src/main/java/com/google/bazel/example/android/activities/MainActivity.java</code></td> -</tr> -<tr> -<td>Resource file directory</td> -<td><code>src/main/java/com/google/bazel/example/android/res/</code></td> -</tr> -</tbody> -</table> - -Note that you're just looking at these files now to become familiar with the -structure of the app. You don't have to edit any of the source files to complete -this tutorial. - -## Update the WORKSPACE file - -Bazel needs to run the Android SDK -[build tools](https://developer.android.com/tools/revisions/build-tools.html) -and uses the SDK libraries to build the app. This means that you need to add -some information to your `WORKSPACE` file so that Bazel knows where to find -them. Note that this step is not required when you build for other platforms. -For example, Bazel automatically detects the location of Java, C++ and -Objective-C compilers from settings in your environment. - -Add the following lines to your `WORKSPACE` file: - -```python -android_sdk_repository( - name = "androidsdk", - # Replace with your installed Android SDK API level - api_level = 25 -) -``` - -This will use the Android SDK specified referenced by the `ANDROID_HOME` -environment variable, and automatically detect the latest build tools -version installed within that location. - -Alternatively, you can explicitly specify the location of the Android -SDK and build tools version to use by including the `path` and -`build_tools_version` attributes: - -```python -android_sdk_repository( - name = "androidsdk", - path = "/path/to/Android/sdk", - api_level = 25, - build_tools_version = "25.0.1" -) -``` - -**Optional:** This is not required by this tutorial, but if you want to compile -native code into your Android app, you also need to download the -[Android NDK](https://developer.android.com/ndk/downloads/index.html) and -tell Bazel where to find it by adding the following rule to your `WORKSPACE` -file: - -```python -android_ndk_repository( - name = "androidndk", - # Replace with the Android NDK API level - api_level = 21 -) -``` - -`api_level` is the version of the Android API the SDK and the NDK target -(for example, 19 for Android K and 21 for Android L). It's not necessary to set -the API levels to the same value for the SDK and NDK. -[This web page](https://developer.android.com/ndk/guides/stable_apis.html) -contains a map from Android releases to NDK-supported API levels. - -Similar to `android_sdk_repository`, the path to the Android NDK is inferred from -the `ANDROID_NDK_HOME` environment variable by default. The path can also be -explicitly specified with a `path` attribute on `android_ndk_repository`. - -## Create a BUILD file - -A [`BUILD` file](/docs/build-ref.html#BUILD_files) is a text file that describes -the relationship between a set of build outputs -- for example, compiled -software libraries or executables -- and their dependencies. These dependencies -may be source files in your workspace or other build outputs. `BUILD` files are -written in the Bazel *build language*. - -`BUILD` files are part of concept in Bazel known as the *package hierarchy*. -The package hierarchy is a logical structure that overlays the directory -structure in your workspace. Each [package](/docs/build-ref.html#packages) is a -directory (and its subdirectories) that contains a related set of source files -and a `BUILD` file. The package also includes any subdirectories, excluding -those that contain their own `BUILD` file. The *package name* is the name of the -directory where the `BUILD` file is located. - -Note that this package hierarchy is distinct from, but coexists with, the Java -package hierarchy for your Android app. - -For the simple Android app in this tutorial, we'll consider all the source files -in `$WORKSPACE/android/` to comprise a single Bazel package. A more complex -project may have many nested packages. - -At a command-line prompt, open your new `BUILD` file for editing: - -```bash -vi $WORKSPACE/android/BUILD -``` - -### Add an android_library rule - -A `BUILD` file contains several different types of instructions for Bazel. The -most important type is the [build rule](/docs/build-ref.html#funcs), which tells -Bazel how to build an intermediate or final software output from a set of source -files or other dependencies. - -Bazel provides two build rules, `android_library` and `android_binary`, that you -can use to build an Android app. For this tutorial, you'll first use the -[`android_library`](/docs/be/android.html#android_library) rule to tell -Bazel how to build an -[Android library module](http://developer.android.com/tools/projects/index.html#LibraryProjects) -from the app source code and resource files. Then you'll use the -`android_binary` rule to tell it how to build the Android application package. - -Add the following to your `BUILD` file: - -```python -android_library( - name = "activities", - srcs = glob(["src/main/java/com/google/bazel/example/android/activities/*.java"]), - custom_package = "com.google.bazel.example.android.activities", - manifest = "src/main/java/com/google/bazel/example/android/activities/AndroidManifest.xml", - resource_files = glob(["src/main/java/com/google/bazel/example/android/activities/res/**"]), -) -``` - -As you can see, the `android_library` build rule contains a set of attributes -that specify the information that Bazel needs to build a library module from the -source files. Note also that the name of the rule is `activities`. You'll -reference the rule using this name as a dependency in the `android_binary` rule. - -### Add an android_binary rule - -The [`android_binary`](/docs/be/android.html#android_binary) rule builds -the Android application package (`.apk` file) for your app. - -Add the following to your build file: - -```python -android_binary( - name = "android", - custom_package = "com.google.bazel.example.android", - manifest = "src/main/java/com/google/bazel/example/android/AndroidManifest.xml", - resource_files = glob(["src/main/java/com/google/bazel/example/android/res/**"]), - deps = [":activities"], -) -``` - -Here, the `deps` attribute references the output of the `activities` rule you -added to the `BUILD` file above. This means that, when Bazel builds the output -of this rule, it checks first to see if the output of the `activities` library -rule has been built and is up-to-date. If not, it builds it and then uses that -output to build the application package file. - -Now, save and close the file. You can compare your `BUILD` file to the -[completed example](https://github.com/bazelbuild/examples/blob/master/tutorial/android/BUILD) -in the `master` branch of the GitHub repo. - -## Run the build - -You use the -[`bazel`](/docs/bazel-user-manual.html) command-line tool to run builds, execute -unit tests and perform other operations in Bazel. This tool is located in the -`output` subdirectory of the location where you installed Bazel. During -[installation](/docs/install.md), you probably added this location to your -path. - -Before you build the sample app, make sure that your current working directory -is inside your Bazel workspace: - -```bash -cd $WORKSPACE -``` - -Now, enter the following to build the sample app: - -```bash -bazel build //android:android -``` - -The [`build`](/docs/bazel-user-manual.html#build) subcommand instructs Bazel to -build the target that follows. The target is specified as the name of a build -rule inside a `BUILD` file, with along with the package path relative to -your workspace directory. Note that you can sometimes omit the package path -or target name, depending on your current working directory at the command -line and the name of the target. See [Labels](/docs/build-ref.html#labels) in -*Bazel Concepts and Terminology* page for more information about target labels -and paths. - -Bazel now launches and builds the sample app. During the build process, its -output will appear similar to the following: - -```bash -INFO: Found 1 target... -Target //android:android up-to-date: - bazel-bin/android/android_deploy.jar - bazel-bin/android/android_unsigned.apk - bazel-bin/android/android.apk -INFO: Elapsed time: 7.237s, Critical Path: 5.81s -``` - -## Find the build outputs - -Bazel stores the outputs of both intermediate and final build operations in -a set of per-user, per-workspace output directories. These directories are -symlinked from the following locations: - -* `$WORKSPACE/bazel-bin`, which stores binary executables and other runnable - build outputs -* `$WORKSPACE/bazel-genfiles`, which stores intermediary source files that are - generated by Bazel rules -* `$WORKSPACE/bazel-out`, which stores other types of build outputs - -Bazel stores the Android `.apk` file generated using the `android_binary` rule -in the `bazel-bin/android/` directory, where the subdirectory name `android` is -derived from the name of the Bazel package. - -At a command prompt, list the contents of this directory and find the -`android.apk` file: - -```bash -ls $WORKSPACE/bazel-bin/android -``` - -## Run the app - -You can now deploy the app to a connected Android device or emulator from the -command line using the -[`bazel mobile-install`](http://bazel.build/docs/bazel-user-manual.html#mobile-install) -command. This command uses the Android Debug Bridge (`adb`) to communicate with -the device. You must set up your device to use `adb` following the instructions -in -[Android Debug Bridge](http://developer.android.com/tools/help/adb.html) before -deployment. - -Enter the following: - -```bash -bazel mobile-install //android:android -``` - -Note that the `mobile-install` subcommand also supports the -[`--incremental`](http://bazel.build/docs/bazel-user-manual.html#mobile-install) -flag that can be used to deploy only those parts of the app that have changed -since the last deployment. - -## What's next - -Now that you've built a sample app for Android, it's time to do the same for -the [iOS app](ios-app.md). diff --git a/site/versions/master/docs/tutorial/app.md b/site/versions/master/docs/tutorial/app.md deleted file mode 100644 index b3b274c2d5..0000000000 --- a/site/versions/master/docs/tutorial/app.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -layout: documentation -title: Build Mobile Application ---- - -# Build Mobile Application - -You can use Bazel to build a variety of software outputs, including -Linux and macOS (OS X) applications written in Java, C++ and Objective-C. You can -also use Bazel to build software for other platforms or written in other -languages. - -This tutorial shows how to use Bazel to build the following: - -* An Android app -* An iOS app -* A mobile backend server running on App Engine - -In this tutorial, you'll learn how to: - -* Set up a Bazel workspace and create a `WORKSPACE` file -* Create `BUILD` files that contain the instructions used by Bazel to build - the software -* Run builds using the Bazel command line tool - -## Requirements - -You can follow the steps in this tutorial on either a Linux or Mac OS X system. -However, you can only build the iOS app if you are running Bazel on OS X. If -you are using Linux, you can skip the iOS instructions and still complete -the rest of the tutorial steps. - -## Sample project - -You don't have to write your own mobile apps and backend server to use this -tutorial. Instead, you'll use a sample project hosted on GitHub. The sample -project is hosted at the following location: - -[https://github.com/bazelbuild/examples/](https://github.com/bazelbuild/examples/) - -You'll grab the sample project files in the next step in this tutorial. - -## What's next - -Let's start off by [setting up](environment.md) the tutorial environment. diff --git a/site/versions/master/docs/tutorial/backend-server.md b/site/versions/master/docs/tutorial/backend-server.md deleted file mode 100644 index 00c61ac43d..0000000000 --- a/site/versions/master/docs/tutorial/backend-server.md +++ /dev/null @@ -1,237 +0,0 @@ ---- -layout: documentation -title: Tutorial - Build the Backend Server ---- - -# Tutorial - Build the Backend Server - -The backend server is a simple web application that runs on Google App Engine -and responds to incoming HTTP requests from the sample Android and iOS apps. - -Here, you'll do the following: - -* Review the source files for the app -* Update the `WORKSPACE` file -* Create a `BUILD` file -* Run the build -* Find the build outputs -* Run the application on a local development server -* Deploy to Google App Engine - -Bazel provides a set of [App Engine build rules](https://github.com/bazelbuild/rules_appengine) -written using the [Skylark](/docs/skylark/index.html) framework. You'll use -these in the steps below to build the application. - -## Review the source files - -The source files for the backend server are located in `$WORKSPACE/backend/`. - -The key files and directories are: - -<table class="table table-condensed table-striped"> -<thead> -<tr> -<td>Name</td> -<td>Location</td> -</tr> -</thead> -<tbody> -<tr> -<td>Source file directory</td> -<td><code>src/main/java/com/google/bazel/example/app/</code></td> -</tr> -<tr> -<td>Web application metadata directory</td> -<td><code>webapp/WEB-INF/</code></td> -</tr> -</tbody> -</table> - -## Update the WORKSPACE file - -As with the Android app, you must add references to -[external dependencies](http://bazel.build/docs/external.html) to your `WORKSPACE` -file. For the backend server, these are references to the App Engine SDK, -the Java Servlet SDK and other libraries needed to build the App Engine -applications. - -### Add the App Engine rule - -When you built the Android app, you added a reference to the location on your -filesystem where you downloaded and installed the Android SDK. For the -backend server, however, you'll give Bazel instructions for downloading the -required App Engine SDK package from a remote server. This is optional. You -can also download and install the SDK manually on your filesystem and reference -it from that location as described in the -[App Engine rule documentation](https://github.com/bazelbuild/rules_appengine). - -Add the following to your `WORKSPACE` file: - -```python -http_archive( - name = "io_bazel_rules_appengine", - sha256 = "f4fb98f31248fca5822a9aec37dc362105e57bc28e17c5611a8b99f1d94b37a4", - strip_prefix = "rules_appengine-0.0.6", - url = "https://github.com/bazelbuild/rules_appengine/archive/0.0.6.tar.gz", -) -load("@io_bazel_rules_appengine//appengine:appengine.bzl", "appengine_repositories") -appengine_repositories() -``` - -[`http_archive`](/docs/be/workspace.html#http_archive) downloads the -AppEngine rules from a GitHub archive. We could also have used -[`git_repository`](/docs/be/workspace.html#git_repository) to fetch the rules -directly from the Git repository. -Then the next two lines use the `appengine_repositories` function defined in -these rules to download the libraries and SDK needed to build AppEngine -applications. - -Now, save and close the file. You can compare your `WORKSPACE` file to the -[completed example](https://github.com/bazelbuild/examples//blob/master/tutorial/WORKSPACE) -in the `master` branch of the GitHub repo. - -## Create a BUILD file - -Now that you have set up the external dependencies, you can go ahead and create -the `BUILD` file for the backend server, as you did previously for the sample -Android and iOS apps. - -Open your new `BUILD` file for editing: - -```bash -vi $WORKSPACE/backend/BUILD -``` - -### Add a java_binary rule - -Add the following to your `BUILD` file: - -```python -java_binary( - name = "app", - srcs = glob(["src/main/java/**/*.java"]), - main_class = "does.not.exist", - deps = [ - "@io_bazel_rules_appengine//appengine:javax.servlet.api", - ], -) -``` - -The [`java_binary`](/docs/be/java.html#java_binary) tells Bazel -how to build a Java `.jar` library for your application, plus a wrapper shell -script that launches the application code from the specified main class. Here, -we're using this rule instead of the -[`java_library`](/docs/be/java.html#java_library) because we need -the `.jar` file to contain all the dependencies required to build the final -App Engine `.war` file. For this reason, we specify a bogus class name -for the `main_class` attribute. - -### Add an appengine_war rule - -Add the following to your `BUILD` file: - -```python -load("@io_bazel_rules_appengine//appengine:appengine.bzl", "appengine_war") - -appengine_war( - name = "backend", - data = [":webapp"], - data_path = "/backend/webapp", - jars = [":app_deploy.jar"], -) - -filegroup( - name = "webapp", - srcs = glob(["webapp/**/*"]), -) -``` - -The [`appengine_war`](/docs/be/appengine.html#appengine_war) -rule builds the final App Engine `war` file from the library `.jar` file and web -application metadata files in the `webapp` directory. - -Save and close the file. Again, the -[completed example](https://github.com/google/bazel-examples/blob/master/tutorial/backend/BUILD) -is in the `master` branch of the GitHub repo. - -## Run the build - -Make sure that your current working directory is inside your Bazel workspace: - -```bash -cd $WORKSPACE -``` - -Now, enter the following to build the sample app: - -```bash -bazel build //backend:backend -``` - -Bazel now launches and builds the sample app. During the build process, its -output will appear similar to the following: - -```bash -INFO: Found 1 target... -Target //backend:backend up-to-date: - bazel-bin/backend/backend.war - bazel-bin/backend/backend.deploy - bazel-bin/backend/backend -INFO: Elapsed time: 56.867s, Critical Path: 2.72s -``` - -## Find the build outputs - -The `.war` file and other outputs are located in the -`$WORKSPACE/bazel-bin/backend` directory. - -In particular, the `appengine_war` rule generates scripts that you can use to -run your backend locally or deploy it to Google App Engine: - -## Run the application on a local development server - -Here, you'll start a local App Engine development server in your environment and -run your application on it. - -To run the application, enter the following: - -```bash -bazel-bin/backend/backend --port=12345 -``` - -Your application will be available at `http://localhost:12345` - -## Deploy to Google App Engine - -You can also deploy the application to the live App Engine serving -environment on Google Cloud Platform. For this scenario, you must first create -a new Cloud Platform project and App Engine application using the Google Cloud -Platform Console. -Follow [this link](https://console.cloud.google.com/projectselector/appengine/create?lang=java&st=true) -to perform these actions. - -Build the target that allows to deploy to App Engine: - -```bash -bazel build --java_toolchain=@io_bazel_rules_appengine//appengine:jdk7 //backend:backend.deploy -``` - -Then, to deploy the application, enter the following: - -```bash -bazel-bin/backend/backend.deploy <project-id> -``` - -The deployment script prompts you to authorize access to Google Cloud Platform. -After you have authorized access the first time, you can deploy the application -using the `bazel` command and the following rule target: - -```bash -bazel run //backend:backend.deploy <project-id> -``` - -Your application URL will be `http://<project-id>.appspot.com`. - -## What's next - -Now let's [review](review.md) the tutorial steps. diff --git a/site/versions/master/docs/tutorial/cpp.md b/site/versions/master/docs/tutorial/cpp.md deleted file mode 100644 index 23362010ef..0000000000 --- a/site/versions/master/docs/tutorial/cpp.md +++ /dev/null @@ -1,395 +0,0 @@ ---- -layout: documentation -title: Build C++ ---- - -Build C++ -========= - -You can use Bazel to build your C++ application. In this tutorial you'll learn how to: - -* Build your first C++ target -* Use external libraries -* Write and run C++ tests -* Use precompiled libraries - -## Setting up your workspace - -Suppose that you have an existing project in a directory, say, -`~/gitroot/my-project/`. Create an empty file at -`~/gitroot/my-project/WORKSPACE` to show Bazel where your project's root is. -We are going to create a small hello world project with the following directory structure: -{% highlight bash %} -└── my-project - ├── lib - │ ├── BUILD - │ ├── hello-greet.cc - │ └── hello-greet.h - ├── main - │ ├── BUILD - │ ├── hello-time.cc - │ ├── hello-time.h - │ └── hello-world.cc - └── WORKSPACE -{% endhighlight %} - -## Creating source files - -Using the following commands to create the necessary source files: -{% highlight bash %} -# If you're not already there, move to your workspace directory. -cd ~/gitroot/my-project -mkdir ./main -cat > main/hello-world.cc <<'EOF' - -#include "lib/hello-greet.h" -#include "main/hello-time.h" -#include <iostream> -#include <string> - -int main(int argc, char** argv) { - std::string who = "world"; - if (argc > 1) { - who = argv[1]; - } - std::cout << get_greet(who) <<std::endl; - print_localtime(); - return 0; -} -EOF - -cat > main/hello-time.h <<'EOF' - -#ifndef MAIN_HELLO_TIME_H_ -#define MAIN_HELLO_TIME_H_ - -void print_localtime(); - -#endif -EOF - -cat > main/hello-time.cc <<'EOF' - -#include "main/hello-time.h" -#include <ctime> -#include <iostream> - -void print_localtime() { - std::time_t result = std::time(nullptr); - std::cout << std::asctime(std::localtime(&result)); -} -EOF - -mkdir ./lib -cat > lib/hello-greet.h <<'EOF' - -#ifndef LIB_HELLO_GREET_H_ -#define LIB_HELLO_GREET_H_ - -#include <string> - -std::string get_greet(const std::string &thing); - -#endif -EOF - -cat > lib/hello-greet.cc <<'EOF' - -#include "lib/hello-greet.h" -#include <string> - -std::string get_greet(const std::string& who) { - return "Hello " + who; -} -EOF -{% endhighlight %} - -## Adding BUILD files - -As you can see from the source code, `main/hello-world.cc` needs to include both `lib/hello-greet.h` and `main/hello-time.h`. -First we create `lib/BUILD` for hello-greet.cc: - -{% highlight python %} -cc_library( - name = "hello-greet", - srcs = ["hello-greet.cc"], - hdrs = ["hello-greet.h"], - visibility = ["//main:__pkg__"], -) -{% endhighlight %} - -Note that `visibility = ["//main:__pkg__"]` indicates `hello-greet` is visible from `main/BUILD`. -Then we'd create the following `main/BUILD` file: - -{% highlight python %} -cc_library( - name = "hello-time", - srcs = ["hello-time.cc"], - hdrs = ["hello-time.h"], -) - -cc_binary( - name = "hello-world", - srcs = ["hello-world.cc"], - deps = [ - ":hello-time", - "//lib:hello-greet", - ], -) -{% endhighlight %} - -Note when depending on a target in the same package, we can just use `:hello-time`. -When the target is in other package, a full path from root should be used, like `//lib:hello-greet`. - -Now you are ready to build your hello world C++ binary: - -{% highlight bash %} -bazel build main:hello-world -{% endhighlight %} - -This produces the following output: - -{% highlight bash %} -INFO: Found 1 target... -Target //main:hello-world up-to-date: - bazel-bin/main/hello-world -INFO: Elapsed time: 2.869s, Critical Path: 1.00s -{% endhighlight %} - -{% highlight bash %} -./bazel-bin/main/hello-world -{% endhighlight %} - -This produces the following output: - -{% highlight bash %} -Hello world -Thu Jun 23 18:51:46 2016 -{% endhighlight %} - -{% highlight bash %} -./bazel-bin/main/hello-world Bazel -{% endhighlight %} - -This produces the following output: - -{% highlight bash %} -Hello Bazel -Thu Jun 23 18:52:10 2016 -{% endhighlight %} - -Congratulations, you've just built your first Bazel target! - -## Transitive includes - -If a file includes a header, then the file's rule should depend on that header's -library. Conversely, only direct dependencies need to be specified as -dependencies. For example, suppose `sandwich.h` includes `bread.h` and -`bread.h` includes `flour.h`. `sandwich.h` doesn't include `flour.h` (who wants -flour in their sandwich?), so the BUILD file would look like: - -```python -cc_library( - name = "sandwich", - srcs = ["sandwich.cc"], - hdrs = ["sandwich.h"], - deps = [":bread"], -) - -cc_library( - name = "bread", - srcs = ["bread.cc"], - hdrs = ["bread.h"], - deps = [":flour"], -) - -cc_library( - name = "flour", - srcs = ["flour.cc"], - hdrs = ["flour.h"], -) -``` - -Here, the `sandwich` library depends on the `bread` library, which depends -on the `flour` library. - -## Adding include paths - -Sometimes you cannot (or do not want to) base include paths at the workspace -root. Existing libraries might already have a include directory that doesn't -match its path in your workspace. For example, suppose you have the following -directory structure: - -``` -└── my-project - ├── third_party - │ └── some_lib - │ ├── BUILD - │ ├── include - │ │ └── some_lib.h - │ └── some_lib.cc - └── WORKSPACE -``` - -Bazel will expect `some_lib.h` to be included as -`third_party/some_lib/include/some_lib.h`, but suppose `some_lib.cc` includes -`"include/some_lib.h"`. To make that include path valid, -`third_party/some_lib/BUILD` will need to specify that the `some_lib/` -directory is an include directory: - -```python -cc_library( - name = "some_lib", - srcs = ["some_lib.cc"], - hdrs = ["some_lib.h"], - copts = ["-Ithird_party/some_lib"], -) -``` - -This is especially useful for external dependencies, as their header files -must otherwise be included with an `external/[repository-name]/` prefix. - -## Including external libraries - -Suppose you are using [Google Test](https://github.com/google/googletest). You -can use one of the `new_` repository functions in the `WORKSPACE` file to -download Google Test and make it available in your repository: - -```python -new_http_archive( - name = "gtest", - url = "https://github.com/google/googletest/archive/release-1.7.0.zip", - sha256 = "b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0", - build_file = "gtest.BUILD", -) -``` - -Then create `gtest.BUILD`, a BUILD file to use to compile Google Test. -Google Test has several "special" requirements that make its `cc_library` rule -more complicated: - -* `googletest-release-1.7.0/src/gtest-all.cc` `#include`s all of the other files in - `googletest-release-1.7.0/src/`, so we need to exclude it from the compile or we'll get - link errors for duplicate symbols. -* It uses header files that are relative to the `googletest-release-1.7.0/include/` directory - (`"gtest/gtest.h"`), so we must add that directory to the include paths. -* It needs to link in pthread, so we add that as a `linkopt`. - -The final rule looks like this: - -```python -cc_library( - name = "main", - srcs = glob( - ["googletest-release-1.7.0/src/*.cc"], - exclude = ["googletest-release-1.7.0/src/gtest-all.cc"] - ), - hdrs = glob([ - "googletest-release-1.7.0/include/**/*.h", - "googletest-release-1.7.0/src/*.h" - ]), - copts = [ - "-Iexternal/gtest/googletest-release-1.7.0/include" - ], - linkopts = ["-pthread"], - visibility = ["//visibility:public"], -) -``` - -This is somewhat messy: everything is prefixed with googletest-release-1.7.0 as a byproduct -of the archive's structure. You can make `new_http_archive` strip this prefix by -adding the `strip_prefix` attribute: - -```python -new_http_archive( - name = "gtest", - url = "https://github.com/google/googletest/archive/release-1.7.0.zip", - sha256 = "b58cb7547a28b2c718d1e38aee18a3659c9e3ff52440297e965f5edffe34b6d0", - build_file = "gtest.BUILD", - strip_prefix = "googletest-release-1.7.0", -) -``` - -Then `gtest.BUILD` would look like this: - -```python -cc_library( - name = "main", - srcs = glob( - ["src/*.cc"], - exclude = ["src/gtest-all.cc"] - ), - hdrs = glob([ - "include/**/*.h", - "src/*.h" - ]), - copts = ["-Iexternal/gtest/include"], - linkopts = ["-pthread"], - visibility = ["//visibility:public"], -) -``` - -Now `cc_` rules can depend on `@gtest//:main`. - -## Writing and running C++ tests - -For example, we could create a test `./test/hello-test.cc` such as: - -```cpp -#include "gtest/gtest.h" -#include "lib/hello-greet.h" - -TEST(HelloTest, GetGreet) { - EXPECT_EQ(get_greet("Bazel"), "Hello Bazel"); -} -``` - -Then create `./test/BUILD` file for your tests: - -```python -cc_test( - name = "hello-test", - srcs = ["hello-test.cc"], - copts = ["-Iexternal/gtest/include"], - deps = [ - "@gtest//:main", - "//lib:hello-greet", - ], -) -``` - -Note in order to make `hello-greet` visible to `hello-test`, we have to add `"//test:__pkg__",` to `visibility` attribute in `./lib/BUILD`. - -Now you can use `bazel test` to run the test. - -{% highlight bash %} -bazel test test:hello-test -{% endhighlight %} - -This produces the following output: - -{% highlight bash %} -INFO: Found 1 test target... -Target //test:hello-test up-to-date: - bazel-bin/test/hello-test -INFO: Elapsed time: 4.497s, Critical Path: 2.53s -//test:hello-test PASSED in 0.3s - -Executed 1 out of 1 tests: 1 test passes. -{% endhighlight %} - - -## Adding dependencies on precompiled libraries - -If you want to use a library that you only have a compiled version of (e.g., -headers and a .so) wrap it in a `cc_library` rule: - -```python -cc_library( - name = "mylib", - srcs = ["mylib.so"], - hdrs = ["mylib.h"], -) -``` - -Then other C++ targets in your workspace can depend on this rule. diff --git a/site/versions/master/docs/tutorial/environment.md b/site/versions/master/docs/tutorial/environment.md deleted file mode 100644 index 68548dfdfc..0000000000 --- a/site/versions/master/docs/tutorial/environment.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -layout: documentation -title: Tutorial - Set Up Your Environment ---- - -# Tutorial - Set Up Your Environment - -The first step in this tutorial is to set up your environment. - -Here, you'll do the following: - -* Install Bazel -* Install Android Studio and the Android SDK -* Install Xcode (macOS (OS X) only) -* Get the sample project from the GitHub repo - -## Install Bazel - -Follow the [installation instructions](/docs/install.md) to install Bazel and -its dependencies. - -## Install the Android SDK tools - -Do the following: - -1. Download and install the - [Android SDK Tools](https://developer.android.com/sdk/index.html#Other). - -2. Run the Android SDK Manager and install the following packages: - - <table class="table table-condensed table-striped"> - <thead> - <tr> - <td>Package</td> - <td>SDK directory</td> - </tr> - </thead> - <tbody> - <tr> - <td>Android SDK Platform Tools</td> - <td><code>platform-tools</code></td> - </tr> - <tr> - <td>Android SDK Build Tools</td> - <td><code>build-tools</code></td> - </tr> - <tr> - <td>Android SDK Platform</td> - <td><code>platform</code></td> - </tr> - </tbody> - </table> - - The SDK Manager is an executable named `android` located in the `tools` - directory. - -## Install Xcode (OS X only) - -If you are following the steps in this tutorial on Mac OS X, download and -install [Xcode](https://developer.apple.com/xcode/downloads/). The Xcode -download contains the iOS libraries, Objective-C compiler other tools -required by Bazel to build the iOS app. - -## Get the sample project - -You also need to get the sample project for the tutorial from GitHub: - -[https://github.com/bazelbuild/examples/](https://github.com/bazelbuild/examples/) - -The GitHub repo has two branches: `source-only` and `master`. The `source-only` -branch contains the source files for the project only. You'll use the files in -this branch in this tutorial. The `master` branch contains both the source files -and completed Bazel `WORKSPACE` and `BUILD` files. You can use the files in this -branch to check your work when you've completed the tutorial steps. - -Enter the following at the command line to get the files in the `source-only` -branch: - -```bash -cd $HOME -git clone -b source-only https://github.com/bazelbuild/examples -``` - -The `git clone` command creates a directory named `$HOME/examples/`. This -directory contains several sample projects for Bazel. The project files for this -tutorial are in `$HOME/examples/tutorial`. - -## What's next - -Now that you have set up your environment, you can -[set up a Bazel workspace](workspace.md). diff --git a/site/versions/master/docs/tutorial/ios-app.md b/site/versions/master/docs/tutorial/ios-app.md deleted file mode 100644 index abf49d27c4..0000000000 --- a/site/versions/master/docs/tutorial/ios-app.md +++ /dev/null @@ -1,207 +0,0 @@ ---- -layout: documentation -title: Tutorial - Build an iOS App ---- - -# Tutorial - Build an iOS App - -Like the [Android app](android-app.md) you built in the previous step, the iOS -app is a simple mobile app that communicates with the -[backend server](backend-server.md). - -Here, you'll do the following: - -* Review the source files for the app -* Create a `BUILD` file -* Build the app for the simulator -* Find the build outputs -* Run/Debug the app on the simulator -* Build the app for a device -* Install the app on a device - -Note that, unlike with the Android app, you don't have to modify your -`WORKSPACE` file to add iOS-specific external dependencies. - -If you're following the steps in this tutorial on macOS (OS X), you can go ahead -and build the sample iOS app as described below. If you are on Linux, skip ahead -to the [next step](backend-server.md). - -## Review the source files - -Let's take a look at the source files for the app. These are located in -`$WORKSPACE/ios-app/UrlGet`. Again, you're just looking at these files now to -become familiar with the structure of the app. You don't have to edit any of the -source files to complete this tutorial. - -## Create a BUILD file - -At a command-line prompt, open your new `BUILD` file for editing: - -```bash -vi $WORKSPACE/ios-app/BUILD -``` - -## Add an objc_library rule - -Bazel provides several build rules that you can use to build an app for the -iOS platform. For this tutorial, you'll first use the -[`objc_library`](/docs/be/objective-c.html#objc_library) rule to tell Bazel -how to build an -[static library](https://developer.apple.com/library/ios/technotes/iOSStaticLibraries/Introduction.html) -from the app source code and Xib files. Then you'll use the -`objc_binary` rule to tell it how to bundle the iOS application. (Note that -this is a minimal use case of the Objective-C rules in Bazel. For example, you -have to use the `ios_application` rule to build multi-architecture iOS -apps.) - -Add the following to your `BUILD` file: - -```python -objc_library( - name = "UrlGetClasses", - srcs = [ - "UrlGet/AppDelegate.m", - "UrlGet/UrlGetViewController.m", - ], - hdrs = glob(["UrlGet/*.h"]), - xibs = ["UrlGet/UrlGetViewController.xib"], -) -``` - -Note the name of the rule, `UrlGetClasses`. - -## Add an objc_binary rule - -The [`objc_binary`](/docs/be/objective-c.html#objc_binary) rule creates a -binary to be bundled in the application. - -Add the following to your `BUILD` file: - -```python -objc_binary( - name = "ios-app-binary", - srcs = [ - "UrlGet/main.m", - ], - deps = [ - ":UrlGetClasses", - ], -) - -``` -Note how the `deps` attribute references the output of the -`UrlGetClasses` rule you added to the `BUILD` file above. - -## Add an ios_application rule - -The [`ios_application`](/docs/be/objective-c.html#ios_application) rule -creates the bundled `.ipa` archive file for the application and also generates -an Xcode project file. - -Add the following to your `BUILD` file: - -```python -ios_application( - name = "ios-app", - binary = ":ios-app-binary", - infoplist = "UrlGet/UrlGet-Info.plist", -) -``` - -Now, save and close the file. You can compare your `BUILD` file to the -[completed example](https://github.com/bazelbuild/examples/blob/master/tutorial/ios-app/BUILD) -in the `master` branch of the GitHub repo. - -## Build the app for the simulator - -Make sure that your current working directory is inside your Bazel workspace: - -```bash -cd $WORKSPACE -``` - -Now, enter the following to build the sample app: - -```bash -bazel build //ios-app:ios-app -``` - -Bazel now launches and builds the sample app. During the build process, its -output will appear similar to the following: - -```bash -INFO: Found 1 target... -Target //ios-app:ios-app up-to-date: - bazel-bin/ios-app/ios-app.ipa - bazel-bin/ios-app/ios-app.xcodeproj/project.pbxproj -INFO: Elapsed time: 3.765s, Critical Path: 3.44s -``` - -## Find the build outputs - -The `.ipa` file and other outputs are located in the -`$WORKSPACE/bazel-bin/ios-app` directory. - -## Run/Debug the app on the simulator - -You can now run the app from Xcode using the iOS Simulator. To run the app, -open the project directory `$WORKSPACE/bazel-bin/ios-app/ios-app.xcodeproj` in -Xcode, choose an iOS Simulator as the runtime scheme and then click the **Run** -button. - -**Note:** If you change anything about the project file set in Xcode (for -example, if you add or remove a file, or add or change a dependency), you must -rebuild the app using Bazel and then re-open the project. - -## Build the app for a device - -You need to set up bazel so that it can find the appropriate provisioning -profile for the device you want to build for. To set up the "default" -provisioning profile for all bazel builds: - - 1. Go to [Apple Profiles](https://developer.apple.com/account/ios/profile/profileList.action) - and download the appropriate provisioning profile for your device. - If this is confusing, please refer to [Apple's documentation](https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/MaintainingProfiles/MaintainingProfiles.html). - 1. Move your profile into `$WORKSPACE/tools/objc`. - 1. Optional - You may want to add your profile to your `.gitignore`. - 1. Edit `$WORKSPACE/tools/objc/BUILD` and add: - - ```python - filegroup( - name = "default_provisioning_profile", - srcs = ["<NAME OF YOUR PROFILE>.mobileprovision"], - ) - ``` - -Now you should be able to build the app for your device: - -```bash -bazel build //ios-app:ios-app --ios_multi_cpus=armv7,arm64 -``` - -This will build the app "fat". If you would prefer just to build for -your specific device architecture you can designate a single architecture. - -If you would like to select a specific Xcode version you can do so -with the `--xcode_version=7.2` option. If for some reason you need to specify -a specific SDK version you can use the `--ios_sdk_version=9.2` option, but the -`--xcode_version` should be sufficient in most circumstances. - -If you would like to specify a minimum version of iOS to run against, you can -do so with the `--ios_minimum_os=7.0` option. - -## Install the app on a device - -The easiest way to install the app on the device is to launch Xcode and use the -`Windows > Devices` command. Select your plugged in device from the list on the -left, and then add the app by clicking on the "plus" sign under installed apps -and selecting the `.ipa` that you built. - -If your app does not launch, please make sure that your device was on your -provisioning profile. The `View Device Logs` button on the `Devices` screen in -Xcode may provide other information as to what has gone wrong. - -## What's next - -The next step is to build a [backend server](backend-server.md) for the two -mobile apps you built in this tutorial. diff --git a/site/versions/master/docs/tutorial/java.md b/site/versions/master/docs/tutorial/java.md deleted file mode 100644 index a916aeb5be..0000000000 --- a/site/versions/master/docs/tutorial/java.md +++ /dev/null @@ -1,576 +0,0 @@ ---- -layout: documentation -title: Introduction to Bazel ---- - -Introduction to Bazel: Build Java -========== - -This tutorial is an introduction for anyone getting started with Bazel. It -focuses on the concepts, setup, and use of Bazel using a Java sample project. - -Estimated time: 30 min - -## What you will learn - -In this tutorial you'll learn how to: - -* Build a target from source files -* Produce a visual representation of the dependency graph -* Break a monolithic binary into smaller libraries -* Use multiple Bazel packages -* Control the visibility of a target between packages -* Use labels to reference a target -* Deploy your target - -## Before you begin - -* [Install Bazel](/docs/install.md) - -## Create the sample Java project - -The first step in this tutorial is to create a small Java project. Even though -the project is in Java, this tutorial will focus on concepts that are helpful -for using Bazel in any language. - -1. Create the directory `~/my-project/` - -2. Move to this directory: - - ``` - cd ~/my-project - ``` - -3. Create the following directories under `my-project`: - - ``` - mkdir -p src/main/java/com/example - ``` - - Note that path uses conventions specific to Java programs. Programs written - in other languages may have a different workspace path and directory - structure. - -4. In the directory you created, add a file called `Greeting.java` with the - following contents: - - ```java - package com.example; - - public class Greeting { - public static void sayHi() { - System.out.println("Hi!"); - } - } - ``` - -5. Add a second file `ProjectRunner.java` with the following contents: - - ```java - package com.example; - - public class ProjectRunner { - public static void main(String args[]) { - Greeting.sayHi(); - } - } - ``` - -You’ve now created a small Java project. It contains one file that will be -compiled into a library, and another which will be an executable that uses the -library. - -The rest of this tutorial focuses on setting up and using Bazel to build these -source files. - -## Build with Bazel - -### Set up the workspace - -Workspaces are directories that contain the source files for one or more -software projects, as well as a WORKSPACE file and BUILD files that contain -the instructions that Bazel uses to build the software. The workspace may also -contain symbolic links to output directories. - -To define the workspace, create an empty text file at the root of the project -and name it `WORKSPACE`. You now have: `~/my-project/WORKSPACE`. - -This directory and its subdirectories are now part of the same workspace. When -Bazel builds an output, all inputs and dependencies must be in the same -workspace. Anything in different workspaces are independent of each other, -though there are ways to link workspaces that are beyond the scope of this -introduction tutorial. - -If you also do the [C++ tutorial](/docs/tutorial/cpp.md), you’ll notice it uses -the same workspace. Bazel can understand multiple targets in multiple languages -in a single workspace. - -### Create a BUILD file - -Bazel looks for files named `BUILD` which describe how to build the project. - -1. In the `~/my-project` directory, create a file and name it BUILD. This BUILD - file is a sibling of the WORKSPACE file. - - In the BUILD file, you use a declarative language similar to Python to - create instances of Bazel rules. These instances are called *rule targets*. - In Bazel, *targets* are either files or rule targets and they are the - elements in a workspace that you can ask Bazel to build. - - For this project, you’ll use the built-in rule `java_binary`. Bazel's - built-in rules are all documented in the - [Build Encyclopedia](/docs/be/overview.html). You can also create your own - rules using the [Bazel rule extension framework](/docs/skylark/concepts.md). - -2. Add this text to the BUILD file: - - ``` - java_binary( - name = "my-runner", - srcs = glob(["src/main/java/com/example/*.java"]), - main_class = "com.example.ProjectRunner", - ) - ``` -As you can see, the text in the BUILD file doesn’t describe what Bazel does -when it executes this rule target. The rule’s implementation handles the -complexity of how it works (such as the compiler used). - -You can treat the rule as a black box, focusing on what inputs it needs, and -the outputs it produces. This rule builds a Java archive ("jar file") as well -as a wrapper shell script with the same name as the rule target. - -When you’re writing your own BUILD file, go to the -[Build Encyclopedia](/docs/be/overview.html) for a description of what a rule -does and for its list of possible attributes you can define. For example, -here’s the entry for the [java_binary](/docs/be/java.html#java_binary) rule in -the Build Encyclopedia. The Build Encyclopedia has information about all of the -rules that are compiled into Bazel. - -Let’s take a look at the rule target that you added to the BUILD file. - -Each rule instantiation in the BUILD file creates one rule target. Here, you’ve -instantiated the rule `java_binary`, creating the target `my-runner`. - -Different rules will require different attributes, though all must include a -“name” attribute. You use these attributes to explicitly list all of the -target’s dependencies and options. In the target above: - -* `my-runner` is the name of the rule target created - -* `glob(["src/main/java/com/example/*.java"])` includes every file in that - directory that ends with .java (see the Build Encyclopedia for more - information about [globbing](/docs/be/functions.html#glob)) - -* `"com.example.ProjectRunner"` specifies the class that contains the main - method. - -### Build with Bazel - -Now you’re ready to build the Java binary. To do so, you’ll use the command -`bazel build` with the target label `//:my-runner`. You reference targets by -using their label. Label syntax is described later in this tutorial. - -1. Build my-runner by using this command: - - ``` - bazel build //:my-runner - ``` - - You’ll see output similar to: - - ``` - INFO: Found 1 target... - Target //:my-runner up-to-date: - bazel-bin/my-runner.jar - bazel-bin/my-runner - INFO: Elapsed time: 1.021s, Critical Path: 0.83s - ``` - -2. Now execute the file by using this command: - - ``` - bazel-bin/my-runner - ``` - -Congratulations, you've built your first Bazel target! - -Let’s take a look at what you built. In `~/my-project`, Bazel created the -directory `bazel-bin` as well as other directories to store information about -the build. Open this directory to look at the files created during the build -process. These output directories keep the outputs separate from your source -tree. - -### Review the dependency graph - -Bazel requires build dependencies to be explicitly declared in BUILD -files. The build will fail if dependencies are missing, so when a build works -the declared dependencies are accurate. With this explicit information about -dependencies, Bazel creates a build graph and uses it to accurately perform -incremental builds. Our small Java project isn’t too exciting, but let’s check -out its build graph. - -The command `bazel query` retrieves information about the graph and the -relationships between targets. Let’s use it to produce a visual representation -of the build graph. - -1. From the root of the workspace (`my-project`), produce a text description - of the graph by using the command: - - ``` - bazel query --noimplicit_deps 'deps(//:my-runner)' --output graph - ``` - -2. Then, paste the output into Graphviz - ([http://www.webgraphviz.com/](http://www.webgraphviz.com/)) to see the - visual representation. - - The graph for the target my-runner will look like this: - -  - -You can see that `my-runner` depends on the two source files in your Java -project. - -You have now set up the workspace and BUILD file, and used Bazel to build your -project. You have also created a visual representation of the build graph to -see the structure of your build. - -## Refine your Bazel build - -### Add dependencies - -Creating one rule target to build your entire project may be sufficient for -small projects. As projects get larger it's important to break up the build -into self-contained libraries that can be assembled into a final product. -Self-contained libraries mean that everything doesn't need to be rebuilt after -small changes and that Bazel can parallelize more of the build steps. These -self-contained libraries also encourages good code hygiene. - -To break up a project, create a separate rule target for the each subcomponent -and then add the subcomponents as dependencies. For the project in this -tutorial, create a rule target to compile the library, and make the executable -depend on it. - -1. Replace the text in the BUILD file with the text below: - - ``` - java_binary( - name = "my-runner", - srcs = ["src/main/java/com/example/ProjectRunner.java"], - main_class = "com.example.ProjectRunner", - deps = [":greeter"], - ) - - java_library( - name = "greeter", - srcs = ["src/main/java/com/example/Greeting.java"], - ) - ``` - -The new `deps` attribute in `java_binary` tells Bazel that the `greeter` library -will be needed to compile the binary. Rules for many languages support the -`deps` attribute, though the exact semantics of the attribute will vary based -on the language and the type of target. The rule -[java_library](/docs/be/java.html#java_library) compiles sources into -a .jar file. Remember to go to the [Build Encyclopedia](/docs/be/overview.html) -for details about specific rules. - -This BUILD file builds the same files as before, but in a different way: now -Bazel will first build the `greeter` library and then build `my-runner`. - -2. Try building //:my-runner using the command: - - ``` - bazel build //:my-runner - ``` - - You’ll see output similar to: - - ``` - INFO: Found 1 target... - Target //:my-runner up-to-date: - bazel-bin/my-runner.jar - bazel-bin/my-runner - INFO: Elapsed time: 2.454s, Critical Path: 1.58s - ``` - - 3. Execute the file by using this command:: - - ``` - bazel-bin/my-runner - ``` - -If you now edit `ProjectRunner.java` and rebuild `my-runner`, the source file -`Greeting.java` will not be recompiled. When the BUILD file had only the one -target, both source files would be recompiled after any change. - -Looking at the dependency graph, you can see that `my-runner` depends on the -same inputs as it did before, but the structure of the build is different. - -The original dependency graph for `my-runner` looked link this: - - - -The dependency graph for `my-runner` after adding a dependency looks like this: - - - -### Use multiple packages - -For larger projects, you will often be dealing with several directories in your -workspace. You can organize your build process by adding a BUILD file to the -top directory of source files that you want to organize together. A directory -containing a BUILD file is called a package. - -Note that Bazel and Java both have the concept of a package. These are -unrelated to each other, though both are related to the structure of the -directories. - -Let’s build the java project using multiple packages. - -1. First, let’s make the Java project a bit more complex. - - 1. Add the following directory and file: - - ``` - mkdir -p src/main/java/com/example/cmdline - ``` - 2. In the directory cmdline, add the file Runner.java with the following - contents: - - ```java - package com.example.cmdline; - - import com.example.Greeting; - - public class Runner { - public static void main(String args[]) { - Greeting.sayHi(); - } - } - ``` - - Now you have a slightly larger Java project that you can organize with - multiple packages. - -2. In the directory `src/main/java/com/example/cmdline`, add an empty text - file and name it BUILD. The structure of the Java project is now: - - ``` - ├── BUILD - ├── src - │ └── main - │ └── java - │ └── com - │ └── example - │ ├── cmdline - │ │ ├── BUILD - │ │ └── Runner.java - │ ├── Greeting.java - │ └── ProjectRunner.java - └── WORKSPACE - ``` - - Each directory in the workspace can be part of only one package. The - workspace now has two BUILD files, and so has two packages: - - 1. The directory `my-project` and its subdirectories (but not including - subdirectories with their own BUILD file, such as `cmdline`), and - - 2. The directory `cmdline` and any subdirectories. - -3. In the new BUILD file, add the following text: - - ``` - java_binary( - name = "runner", - srcs = ["Runner.java"], - main_class = "com.example.cmdline.Runner", - deps = ["//:greeter"] - ) - ``` - - The file `Runner.java` depends on `com.example.Greeting`. In the BUILD file - this dependency is shown by listing the rule target `greeter` (with the - label `//:greeter`). - - Below is what the dependency graph for runner will look like. You can see - how `//:greeter` gives the dependency on `Greeting.java`. - -  - - -4. However, if you try to build runner right now you'll get a permissions - error. You can see the permission error by trying to build the target using - the command: - - ``` - bazel build //src/main/java/com/example/cmdline:runner - ``` - - By default, rule targets are private, which means that they can only be - depended on by targets in the same BUILD file. This privacy prevents - libraries that are implementation details from leaking into public APIs, - but it also means that you must explicitly allow `runner` to depend on - `//:greeter`. - - -5. Make a rule target visible to rule targets in other BUILD files by adding - a `visibility` attribute. To make the `greeter` rule target in - `~/my-project/BUILD` visible to any rule target in the new package, add the - following visibility attribute: - - ``` - java_library( - name = "greeter", - srcs = ["src/main/java/com/example/Greeting.java"], - visibility = ["//src/main/java/com/example/cmdline:__pkg__"], - ) - ``` - - The target `//:greeter` is now visible to any target in the - `//src/main/java/com/example/cmdline` package. - - See the Build Encyclopedia for more - [visibility options](/docs/be/common-definitions.html#common.visibility). - - -6. Now you can build the runner binary by using the command: - - ``` - bazel build //src/main/java/com/example/cmdline:runner - ``` - - You’ll see output similar to: - - ``` - INFO: Found 1 target... - Target //src/main/java/com/example/cmdline:runner up-to-date: - bazel-bin/src/main/java/com/example/cmdline/runner.jar - bazel-bin/src/main/java/com/example/cmdline/runner - INFO: Elapsed time: 1.576s, Critical Path: 0.81s - ``` - - -7. Execute the file by using this command: - - ``` - bazel-bin/src/main/java/com/example/cmdline/runner - ``` - -You’ve now refined your build so that it is broken down into smaller -self-contained libraries, and so that the explicit dependencies are more -granular. You’ve also built the Java project using multiple packages. - -## Use labels to reference targets - -In the BUILD files and in the command line, you have been using target labels -to reference targets. The label’s syntax is: `//path/to/package:target-name`, -where “`//`” is the workspace’s root, and “`:`” separates the package name and -the target name. If the target is a rule target and so defined in a BUILD file, -“`path/to/package`” would be the path of the BUILD file itself. “`Target-name`” -would be the same as the “`name`” attribute in the target in the BUILD file. - -The first BUILD file you created in this tutorial is in the same directory as -the WORKSPACE file. When referencing rule targets defined in that file, nothing -is needed for the path to the package because the workspace root and the package -root are the same directory. Here are the labels of the two targets defined -in that first BUILD file: - -``` -//:my-runner -//:greeter -``` - -The second BUILD file has a longer path from the workspace root to the package -root. The label for the target in that BUILD file is: - -``` -//src/main/java/com/example/cmdline:runner -``` - -Target labels can be shortened in a variety of ways. Within a BUILD file, if -you’re referencing a target from the same package, you can write the label -starting at “`:`”. For example, the rule target `greeter` can always be written -as `//:greeter`, and in the BUILD file where it’s defined, it can also be -written as `:greeter`. This shortened label in a BUILD file makes it immediately -clear which targets are in the current package. - -A rule target’s name will always be defined by its name attribute. A target’s -name is a bit more complex when it’s in a directory other than the root -of the package. In that case, the target’s label is: -`//path/to/package:path/to/file/file_name`. - -## Package a Java target for deployment - -To understand what you’ve built and what else can be built with Bazel, you need -to understand the capabilities of the rules used in your BUILD files. Always go -to the [Build Encyclopedia](/docs/be/overview.html) for this information. - -Let’s look at packaging a Java target for deployment, which requires you to -know the capabilities of the rule `java_binary`. - -You’re able to run the Java binaries you created in this tutorial, but you -can’t simply run it on a server, because it relies on the greeting library jar -to actually run. "Packaging an artifact so it can be run reliably outside the -development environment involves bundling it with all of its runtime -dependencies. Let's see now what’s needed to package the binaries. - -The rule [java_binary](/docs/be/java.html#java_binary) produces a Java archive -(“jar file”) and a wrapper shell script. The file `<target-name>_deploy.jar` -is suitable for deployment, but it’s only built by this rule if explicitly -requested. Let’s investigate. - -1. Look at the contents of the output `runner.jar` by using this command: - - ``` - jar tf bazel-bin/src/main/java/com/example/cmdline/runner.jar - ``` - - You’ll see output similar to: - - ``` - META-INF/ - META-INF/MANIFEST.MF - com/ - com/example/ - com/example/cmdline/ - com/example/cmdline/Runner.class - ``` - - You can see that `runner.jar` contains `Runner.class`, but not its - dependency `Greeting.class`. The `runner` script that Bazel generates adds - the greeter jar to the classpath, so running this program works locally. It - will not work if you want to copy `runner.jar` to another machine and use - it as a standalone binary. - - -2. The rule `java_binary` allows you to build a self-contained binary that can - be deployed. To create this binary, build `runner_deploy.jar` (or, more - generally, `<target-name>_deploy.jar`) by using this command: - - ``` - bazel build //src/main/java/com/example/cmdline:runner_deploy.jar - ``` - - You’ll see output similar to: - - ``` - INFO: Found 1 target... - Target //src/main/java/com/example/cmdline:runner_deploy.jar up-to-date: - bazel-bin/src/main/java/com/example/cmdline/runner_deploy.jar - INFO: Elapsed time: 1.700s, Critical Path: 0.23s - ``` - - The file runner_deploy.jar will contain all of its dependencies, and so can - be used as a standalone binary. - -You’ve now created a Java target that you can distribute and deploy. To do so, -you had to be aware of what outputs the Bazel Java rule `java_binary` is able to -produce. - -## Further topics - -* Try the tutorial [Build C++](/docs/tutorial/cpp.md). -* Try the tutorial [Build Mobile Application](/docs/tutorial/app.md). - diff --git a/site/versions/master/docs/tutorial/review.md b/site/versions/master/docs/tutorial/review.md deleted file mode 100644 index 62d4501ad4..0000000000 --- a/site/versions/master/docs/tutorial/review.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -layout: documentation -title: Tutorial - Review ---- - -# Tutorial - Review - -In this tutorial, you used Bazel to build an [Android app](android-app.md), -an [iOS app](ios-app.md) and a [backend server](backend-server.md) that runs on -Google App Engine. - -To build these software outputs, you: - -* Set up a Bazel [workspace](workspace.md) that contained the source code - for the components and a `WORKSPACE` that identifies the top level of the - workspace directory -* Created a `BUILD` file for each component -* Updated the `WORKSPACE` file to contain references to the required - external dependencies -* Ran Bazel to build the software components - -The built mobile apps and backend server application files are located in the -`$WORKSPACE/bazel-bin` directory. - -Note that completed `WORKSPACE` and `BUILD` files for this tutorial are located -in the -[master branch](https://github.com/bazelbuild/examples/tree/master/tutorial) -of the GitHub repo. You can compare your work to the completed files for -additional help or troubleshooting. diff --git a/site/versions/master/docs/tutorial/workspace.md b/site/versions/master/docs/tutorial/workspace.md deleted file mode 100644 index eee1b1a350..0000000000 --- a/site/versions/master/docs/tutorial/workspace.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -layout: documentation -title: Tutorial - Set Up a Workspace ---- - -# Tutorial - Set Up a Workspace - -A [workspace](/docs/build-ref.html#workspaces) is a directory that contains the -source files for one or more software projects, as well as a `WORKSPACE` file -and `BUILD` files that contain the instructions that Bazel uses to build -the software. The workspace may also contain symbolic links to output -directories. - -A workspace directory can be located anywhere on your filesystem. In this -tutorial, your workspace directory is `$HOME/examples/tutorial/`, which -contains the sample project files you cloned from the GitHub repo in the -previous step. - -Note that Bazel itself doesn't make any requirements about how you organize -source files in your workspace. The sample source files in this tutorial are -organized according to common conventions for Android apps, iOS apps and App -Engine applications. - -For your convenience, set the `$WORKSPACE` environment variable now to refer to -your workspace directory. At the command line, enter: - -```bash -export WORKSPACE=$HOME/examples/tutorial -``` - -## Create a WORKSPACE file - -Every workspace must have a text file named `WORKSPACE` located in the top-level -workspace directory. This file may be empty or it may contain references -to [external dependencies](/docs/external.html) required to build the -software. - -For now, you'll create an empty `WORKSPACE` file, which simply serves to -identify the workspace directory. In later steps, you'll update the file to add -external dependency information. - -Enter the following at the command line: - -```bash -touch $WORKSPACE/WORKSPACE -``` - -This creates the empty `WORKSPACE` file. - -## What's next - -Now that you've set up your workspace, you can -[build the Android app](android-app.md). diff --git a/site/versions/master/docs/windows.md b/site/versions/master/docs/windows.md deleted file mode 100644 index 5a0b6b51c4..0000000000 --- a/site/versions/master/docs/windows.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -layout: documentation -title: Windows ---- - -# Using Bazel on Windows - -Windows support is experimental. Known issues are [marked with label -"Windows"](https://github.com/bazelbuild/bazel/issues?q=is%3Aissue+is%3Aopen+label%3A%22category%3A+multi-platform+%3E+windows%22) -on github issues. - -We currently support only 64 bit Windows 7 or higher and we compile Bazel as a -msys2 binary. - -## <a name="install"></a>Installation - -See instructions on the [installation page](install-windows.md). - -## <a name="requirements"></a>Requirements - -Before you can compile or run Bazel, you will need to set some environment -variables: - -```bash -export JAVA_HOME="$(ls -d C:/Program\ Files/Java/jdk* | sort | tail -n 1)" -export BAZEL_SH=c:/tools/msys64/usr/bin/bash.exe -``` - -If you run outside of `bash`, ensure that ``msys-2.0.dll`` is in your ``PATH`` -(if you install msys2 to ``c:\tools\msys64``, just add -``c:\tools\msys64\usr\bin`` to ``PATH``). - -If you have another tool that vendors msys2 (such as msysgit), then -``c:\tools\msys64\usr\bin`` must appear in your ``PATH`` *before* entries for -those tools. - -Similarly, if you have [bash on Ubuntu on -Windows](https://msdn.microsoft.com/en-gb/commandline/wsl/about) installed, you -should make sure ``c:\tools\msys64\usr\bin`` appears in ``PATH`` *before* -``c:\windows\system32``, because otherwise Windows' ``bash.exe`` is used before -msys2's. - -Use ``where msys-2.0.dll`` to ensure your ``PATH`` is set up correctly. - -To **run** Bazel (even pre-built binaries), you will need: - -* Java JDK 8 or later -* [msys2 shell](https://msys2.github.io/) (need to be installed at - ``C:\tools\msys64\``). - * We build against version - [20160205](https://sourceforge.net/projects/msys2/files/Base/x86_64/msys2-x86_64-20160205.exe/download), - you will need this version in order to run the pre-built - [release Bazel binaries](https://github.com/bazelbuild/bazel/releases). - * You can also use newer versions or the - [latest version](https://sourceforge.net/projects/msys2/files/latest/download?source=files), - but then you will need to compile Bazel from the distribution archive (the - source zip file) so that it's linked against the right version of - ``msys-2.0.dll``. See also the - [known issues](install-compile-source.md#known-issues-when-compiling-from-source). -* Several msys2 packages. Use the ``pacman`` command to install them: - - ``` - pacman -Syuu gcc git curl zip unzip zlib-devel - ``` - -To **compile** Bazel, in addition to the above you will need: - -* [Visual C++ Build Tools](http://landinghub.visualstudio.com/visual-cpp-build-tools) - or the full [Visual C++](https://www.visualstudio.com/) (as part of Visual - Studio; Community Edition is fine) with Windows SDK installed. -* You may need to apply some patches/workarounds, see the - [known issues](install-compile-source.md#known-issues-when-compiling-from-source). - -## <a name="compiling"></a>Compiling Bazel on Windows - -Ensure you have the [requirements](#requirements). - -To build Bazel: - -* Open the msys2 shell. -* Clone the Bazel git repository as normal. -* Set the environment variables (see above) -* Run ``compile.sh`` in Bazel directory. -* If all works fine, bazel will be built at ``output\bazel.exe``. - - -## <a name="using"></a>Using Bazel on Windows - -Bazel now supports building C++, Java and Python targets on Windows. - -### Build C++ - -To build C++ targets, you will need: - -* [Visual Studio](https://www.visualstudio.com/) -<br/>We are using MSVC as the native C++ toolchain, so please ensure you have Visual -Studio installed with the `Visual C++ > Common Tools for Visual C++` and -`Visual C++ > Microsoft Foundation Classes for C++` features. -(which is NOT the default installation type of Visual Studio). -You can set `BAZEL_VS` environment variable to tell Bazel -where Visual Studio is, otherwise Bazel will try to find the latest version installed. -<br/>For example: `export BAZEL_VS="C:/Program Files (x86)/Microsoft Visual Studio 14.0"` - -* [Python](https://www.python.org/downloads/) -<br/>Both Python 2 and Python 3 are supported. -Currently, we use Python wrapper scripts to call the actual MSVC compiler, so -please make sure Python is installed and its location is added into PATH. -It's also a good idea to set `BAZEL_PYTHON` environment variable to tell Bazel -where Python is. -<br/>For example: `export BAZEL_PYTHON=C:/Python27/python.exe` - -Bazel will auto-configure the location of Visual Studio and Python at the first -time you build any target. -If you need to auto-configure again, just run `bazel clean` then build a target. - -If everything is set up, you can build C++ target now! - -```bash -bazel build examples/cpp:hello-world -./bazel-bin/examples/cpp/hello-world.exe -bazel run examples/cpp:hello-world -``` - -However, with Bazel version prior to 0.5.0, MSVC -toolchain is not default on Windows, you should use flag -`--cpu=x64_windows_msvc` to enable it like this: - -```bash -bazel build --cpu=x64_windows_msvc examples/cpp:hello-world -``` - -### Build Java - -Building Java targets works well on Windows, no special configuration is needed. -Just try: - -```bash -bazel build examples/java-native/src/main/java/com/example/myproject:hello-world -./bazel-bin/examples/java-native/src/main/java/com/example/myproject/hello-world -bazel run examples/java-native/src/main/java/com/example/myproject:hello-world -``` - -### Build Python - -On Windows, we build a self-extracting zip file for executable Python targets, you can even use -`python ./bazel-bin/path/to/target` to run it in native Windows command line (cmd.exe). -See more details in this [design doc](/designs/2016/09/05/build-python-on-windows.html). - -```bash -bazel build examples/py_native:bin -./bazel-bin/examples/py_native/bin -python ./bazel-bin/examples/py_native/bin # This works in both msys and cmd.exe -bazel run examples/py_native:bin -``` diff --git a/site/windows-chocolatey-maintenance.md b/site/windows-chocolatey-maintenance.md deleted file mode 100644 index fbaa93ca6c..0000000000 --- a/site/windows-chocolatey-maintenance.md +++ /dev/null @@ -1,73 +0,0 @@ ---- -layout: contribute -title: Windows Chocolatey maintenance ---- - -# Maintaining Bazel Chocolatey package on Windows - -**NOTE:** The Chocolatey package is experimental; please provide feedback -(`@petemounce` in issue tracker). - -## Prerequisites - -You need: - -* [chocolatey package manager](https://chocolatey.org) installed -* (to publish) a chocolatey API key granting you permission to publish the - `bazel` package - * [@petemounce](https://github.com/petemounce) currently maintains this - unofficial package. -* (to publish) to have set up that API key for the chocolatey source locally - via `choco apikey -k <your key here> -s https://chocolatey.org/` - -## Build - -Compile bazel with msys2 shell and `compile.sh`. - -```powershell -pushd scripts/packages/chocolatey - ./build.ps1 -version 0.3.2 -mode local -popd -``` - -Should result in `scripts/packages/chocolatey/bazel.<version>.nupkg` being -created. - -The `build.ps1` script supports `mode` values `local`, `rc` and `release`. - -## Test - -0. Build the package (with `-mode local`) - - * run a webserver (`python -m SimpleHTTPServer` in - `scripts/packages/chocolatey` is convenient and starts one on - `http://localhost:8000`) - -0. Test the install - - The `test.ps1` should install the package cleanly (and error if it did not - install cleanly), then tell you what to do next. - -0. Test the uninstall - - ```sh - choco uninstall bazel - # should remove bazel from the system - ``` - -Chocolatey's moderation process automates checks here as well. - -## Release - -Modify `tools/parameters.json` for the new release's URI and checksum once the -release has been published to github releases. - -```powershell -./build.ps1 -version <version> -isRelease -./test.ps1 -version <version> -# if the test.ps1 passes -choco push bazel.x.y.z.nupkg --source https://chocolatey.org/ -``` - -Chocolatey.org will then run automated checks and respond to the push via email -to the maintainers. diff --git a/src/test/java/com/google/devtools/build/lib/BUILD b/src/test/java/com/google/devtools/build/lib/BUILD index f073b8c8a0..916a043884 100644 --- a/src/test/java/com/google/devtools/build/lib/BUILD +++ b/src/test/java/com/google/devtools/build/lib/BUILD @@ -805,7 +805,7 @@ java_test( size = "medium", srcs = ["packages/BazelDocumentationTest.java"], data = [ - "//site:versions/master/docs/bazel-user-manual.html", + "//site:docs/bazel-user-manual.html", ], test_class = "com.google.devtools.build.lib.packages.BazelDocumentationTest", deps = [ diff --git a/src/test/java/com/google/devtools/build/lib/packages/BazelDocumentationTest.java b/src/test/java/com/google/devtools/build/lib/packages/BazelDocumentationTest.java index 9e9ddee4c3..271df6a8a7 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/BazelDocumentationTest.java +++ b/src/test/java/com/google/devtools/build/lib/packages/BazelDocumentationTest.java @@ -38,7 +38,7 @@ public class BazelDocumentationTest { */ @Test public void testBazelUserManual() throws Exception { - String documentationFilePath = "site/versions/master/docs/bazel-user-manual.html"; + String documentationFilePath = "site/docs/bazel-user-manual.html"; if (OS.getCurrent() == OS.WINDOWS) { documentationFilePath = WindowsTestUtil.getRunfile("io_bazel/" + documentationFilePath); } |
