diff --git a/build_docs.pl b/build_docs.pl
index 96313b9b50e05..34e72c6f1c795 100755
--- a/build_docs.pl
+++ b/build_docs.pl
@@ -231,6 +231,7 @@ sub build_all {
say "Skipping documentation builds."
}
else {
+ say "Building docs";
build_entries( $build_dir, $temp_dir, $toc, @$contents );
say "Writing main TOC";
diff --git a/integtest/Makefile b/integtest/Makefile
index 97eec8d2c8b88..b9a19dce534ef 100644
--- a/integtest/Makefile
+++ b/integtest/Makefile
@@ -25,6 +25,7 @@ check: \
keep_hash \
sub_dir \
keep_hash_and_sub_dir \
+ multi_branch \
open_all
.PHONY: style
@@ -170,6 +171,7 @@ keep_hash:
--target_repo $(TMP)/dest.git \
--conf $(TMP)/conf.yaml \
--keep_hash | tee $(TMP)/out
+ $(call GREP_V,'Test book',$(TMP)/out)
$(call GREP,'No changes to push',$(TMP)/out)
# We expact the same files as the minimal because we the changes that we
@@ -196,8 +198,10 @@ sub_dir:
/docs_build/build_docs.pl --in_standard_docker --all --push \
--target_repo $(TMP)/dest.git \
--conf $(TMP)/conf.yaml \
- --sub_dir source:master:$(TMP)/to_sub
-
+ --sub_dir source:master:$(TMP)/to_sub | tee $(TMP)/out
+ $(call GREP,'Test book: Building master...',$(TMP)/out)
+ $(call GREP,'Test book: Finished master',$(TMP)/out)
+ $(call GREP,'Test book: Copying master to current',$(TMP)/out)
$(call MINIMAL_ALL_EXPECTED_FILES,'local changes')
.PHONY: keep_hash_and_sub_dir
@@ -214,7 +218,11 @@ keep_hash_and_sub_dir:
sed 's|--tmp--|$(TMP)|' two_repos_conf.yaml > $(TMP)/conf.yaml
/docs_build/build_docs.pl --in_standard_docker --all --push \
--target_repo $(TMP)/dest.git \
- --conf $(TMP)/conf.yaml
+ --conf $(TMP)/conf.yaml | tee $(TMP)/out
+ $(call GREP,'Test book: Building master...',$(TMP)/out)
+ $(call GREP,'Test book: Finished master',$(TMP)/out)
+ $(call GREP,'Test book: Copying master to current',$(TMP)/out)
+ $(call GREP,'Pushing changes',$(TMP)/out)
# Move a "bad" file into source2 so we can be sure we're not picking it up
cp ../README.asciidoc $(TMP)/source2/index.asciidoc
@@ -225,8 +233,9 @@ keep_hash_and_sub_dir:
/docs_build/build_docs.pl --in_standard_docker --all --push \
--target_repo $(TMP)/dest.git \
--conf $(TMP)/conf.yaml \
- --keep_hash | tee /tmp/out
- $(call GREP,'No changes to push',/tmp/out)
+ --keep_hash | tee $(TMP)/out
+ $(call GREP_V,'Test book',$(TMP)/out)
+ $(call GREP,'No changes to push',$(TMP)/out)
# Setup the directory we'd like to substitute
mkdir -p $(TMP)/to_sub/docs
@@ -237,11 +246,84 @@ keep_hash_and_sub_dir:
--target_repo $(TMP)/dest.git \
--conf $(TMP)/conf.yaml \
--keep_hash --sub_dir source1:master:$(TMP)/to_sub | tee $(TMP)/out
+ $(call GREP,'Test book: Building master...',$(TMP)/out)
+ $(call GREP,'Test book: Finished master',$(TMP)/out)
+ $(call GREP,'Test book: Copying master to current',$(TMP)/out)
$(call GREP,'Pushing changes',$(TMP)/out)
git clone $(TMP)/dest.git $(TMP)/dest
$(call GREP,'extra extra extra',$(TMP)/dest/html/test/current/_chapter.html)
+.PHONY: multi_branch
+multi_branch:
+ # Tests for how we rebuild books with multiple branches
+
+ # When the book hasn't been built before we build all branches
+ rm -rf $(TMP)
+ $(call INIT_REPO_WITH_FILE,$(TMP)/source,minimal.asciidoc,docs/index.asciidoc)
+ cd $(TMP)/source && git checkout -b prev
+ git init --bare $(TMP)/dest.git
+
+ sed 's|--tmp--|$(TMP)|' multi_branch.yaml > $(TMP)/conf.yaml
+ /docs_build/build_docs.pl --in_standard_docker --all --push \
+ --target_repo $(TMP)/dest.git \
+ --conf $(TMP)/conf.yaml | tee $(TMP)/out
+ $(call GREP,'Test book: Building master...',$(TMP)/out)
+ $(call GREP,'Test book: Finished master',$(TMP)/out)
+ $(call GREP,'Test book: Building prev...',$(TMP)/out)
+ $(call GREP,'Test book: Finished prev',$(TMP)/out)
+ $(call GREP,'Test book: Copying master to current',$(TMP)/out)
+ $(call GREP,'Test book: Writing versions TOC',$(TMP)/out)
+ $(call GREP,'Pushing changes',$(TMP)/out)
+
+ git clone $(TMP)/dest.git $(TMP)/dest
+ # The main index links to the current version
+ $(call GREP,'Test book \[master\]',$(TMP)/dest/html/index.html)
+ $(call GREP,'other versions',$(TMP)/dest/html/index.html)
+ # And the book's index links to all versions
+ $(call GREP,'Test book: master (current)',$(TMP)/dest/html/test/index.html)
+ $(call GREP,'Test book: prev',$(TMP)/dest/html/test/index.html)
+ # And all versions have been built
+ [ -s $(TMP)/dest/html/test/current/index.html ]
+ [ -s $(TMP)/dest/html/test/master/index.html ]
+ [ -s $(TMP)/dest/html/test/prev/index.html ]
+
+ # When one of the non-current branches has changed we rebuild it but do not
+ # copy the current branch
+ cd $(TMP)/source && \
+ git checkout prev && \
+ echo "changed" >> docs/index.asciidoc && \
+ git add . && \
+ git commit -m "change"
+ /docs_build/build_docs.pl --in_standard_docker --all --push \
+ --target_repo $(TMP)/dest.git \
+ --conf $(TMP)/conf.yaml | tee $(TMP)/out
+ $(call GREP_V,'Test book: Building master...',$(TMP)/out)
+ $(call GREP_V,'Test book: Finished master',$(TMP)/out)
+ $(call GREP,'Test book: Building prev...',$(TMP)/out)
+ $(call GREP,'Test book: Finished prev',$(TMP)/out)
+ $(call GREP_V,'Test book: Copying master to current',$(TMP)/out)
+ $(call GREP,'Test book: Writing versions TOC',$(TMP)/out)
+ $(call GREP,'Pushing changes',$(TMP)/out)
+
+ # When the current branch has changed we rebuild it and copy it to the
+ # current branch
+ cd $(TMP)/source && \
+ git checkout master && \
+ echo "changed" >> docs/index.asciidoc && \
+ git add . && \
+ git commit -m "change"
+ /docs_build/build_docs.pl --in_standard_docker --all --push \
+ --target_repo $(TMP)/dest.git \
+ --conf $(TMP)/conf.yaml | tee $(TMP)/out
+ $(call GREP,'Test book: Building master...',$(TMP)/out)
+ $(call GREP,'Test book: Finished master',$(TMP)/out)
+ $(call GREP_V,'Test book: Building prev...',$(TMP)/out)
+ $(call GREP_V,'Test book: Finished prev',$(TMP)/out)
+ $(call GREP,'Test book: Copying master to current',$(TMP)/out)
+ $(call GREP,'Test book: Writing versions TOC',$(TMP)/out)
+ $(call GREP,'Pushing changes',$(TMP)/out)
+
.PHONY: open_all
open_all:
# Test that `--all --open` starts nginx in a usable way.
@@ -259,21 +341,6 @@ open_all:
$(call GREP,'Location: http://localhost:8000/guide/en/elasticsearch/reference/current/setup.html',$(TMP)/guide/rdir)
kill -QUIT $$(cat /run/nginx/nginx.pid)
-define GREP=
- # grep for a string in a file, outputting the whole file if there isn't
- # a match.
- [ -e $(2) ] || { \
- echo "can't find $(2)"; \
- ls $$(dirname $(2)); \
- false; \
- }
- grep $(1) $(2) > /dev/null || { \
- echo "Couldn't find $(1) in $(2):"; \
- cat $(2); \
- false; \
- }
-endef
-
define SETUP_MINIMAL_ALL=
# First build a repository to use as the source.
$(call INIT_REPO_WITH_FILE,$(TMP)/source,minimal.asciidoc,index.asciidoc)
@@ -292,7 +359,11 @@ define BUILD_MINIMAL_ALL=
$(SETUP_MINIMAL_ALL)
/docs_build/build_docs.pl --in_standard_docker --all --push \
--target_repo $(TMP)/dest.git \
- --conf $(TMP)/conf.yaml
+ --conf $(TMP)/conf.yaml | tee $(TMP)/out
+ $(call GREP,'Test book: Building master...',$(TMP)/out)
+ $(call GREP,'Test book: Finished master',$(TMP)/out)
+ $(call GREP,'Test book: Copying master to current',$(TMP)/out)
+ $(call GREP,'Test book: Writing redirect to current branch...',$(TMP)/out)
endef
define MINIMAL_ALL_EXPECTED_FILES=
@@ -323,13 +394,28 @@ define GREP=
# grep for a string in a file, outputting the whole file if there isn't
# a match.
[ -e $(2) ] || { \
- echo "can't find $(2)"; \
+ echo "can't find \'$(2)\'"; \
ls $$(dirname $(2)); \
false; \
}
- grep $(1) $(2) > /dev/null || { \
- echo "Couldn't" find $(1) in $(2):; \
+ grep -q $(1) $(2) || { \
+ echo "Couldn't" find \'$(1)\' in \'$(2)\':; \
cat $(2); \
false; \
}
endef
+
+define GREP_V=
+ # grep for a string in a file, outputting the whole file if there *is*
+ # a match.
+ [ -e $(2) ] || { \
+ echo "can't find \'$(2)\'"; \
+ ls $$(dirname $(2)); \
+ false; \
+ }
+ grep -qv $(1) $(2) || { \
+ echo Found \'$(1)\' in \'$(2)\':; \
+ cat $(2); \
+ false; \
+ }
+endef
\ No newline at end of file
diff --git a/integtest/multi_branch.yaml b/integtest/multi_branch.yaml
new file mode 100644
index 0000000000000..cdfd1949a325e
--- /dev/null
+++ b/integtest/multi_branch.yaml
@@ -0,0 +1,54 @@
+# This is a small config file for build_docs.pl used for testing
+
+# This block is required even when building a single doc
+template:
+ path: .template/
+ branch:
+ default:
+ base_url: 'https://www.elastic.co/'
+ template_url: 'https://www.elastic.co/guide_template'
+ staging:
+ base_url: 'https://stag-www.elastic.co/'
+ template_url: 'https://stag-www.elastic.co/guide_template'
+ defaults:
+ POSTHEAD: |
+
+ FINAL: |
+
+
+
+paths:
+ # These two configure where to put things in the --target_repo. The
+ # paths are resolved relative to the root of the --target_repo.
+ build: html/
+ branch_tracker: html/branches.yaml
+ # This configures which directory to use for cloning repos. Because
+ # these are big we tend to want them to be in a non-temporary but
+ # .gitignored spot. The path is resolved relative to the root of the
+ # docs repo.
+ repos: --tmp--/repos/
+
+# This configures all of the repositories used to build the docs
+repos:
+ # Normally we use the `https://` prefix to clone from github but this file
+ # is for testing so use a string that we can find with sed and replace with
+ # a file.
+ source: --tmp--/source
+
+# The title to use for the table of contents
+contents_title: Elastic Stack and Product Documentation
+
+# The actual books to build
+contents:
+ -
+ title: Test book
+ prefix: test
+ current: master
+ branches: [ master, prev ]
+ index: docs/index.asciidoc
+ tags: test tag
+ subject: Test
+ sources:
+ -
+ repo: source
+ path: docs/index.asciidoc
diff --git a/lib/ES/Book.pm b/lib/ES/Book.pm
index a11a8123941cb..41df3393f1f22 100644
--- a/lib/ES/Book.pm
+++ b/lib/ES/Book.pm
@@ -156,8 +156,6 @@ sub build {
#===================================
my ( $self, $rebuild ) = @_;
- say "Book: " . $self->title;
-
my $toc = ES::Toc->new( $self->title );
my $dir = $self->dir;
$dir->mkpath;
@@ -173,8 +171,11 @@ sub build {
);
my $latest = 1;
+ my $rebuilding_any_branch = 0;
+ my $rebuilding_current_branch = 0;
for my $branch ( @{ $self->branches } ) {
- $self->_build_book( $branch, $pm, $rebuild, $latest );
+ my $building = $self->_build_book( $branch, $pm, $rebuild, $latest );
+ $rebuilding_any_branch ||= $building;
$latest = 0;
my $branch_title = $self->branch_title($branch);
@@ -184,7 +185,7 @@ sub build {
url => "current/index.html"
}
);
-
+ $rebuilding_current_branch = $building;
}
else {
$toc->add_entry(
@@ -195,12 +196,13 @@ sub build {
}
}
$pm->wait_all_children();
- $self->_copy_branch_to_current( $self->current );
+ $self->_copy_branch_to_current( $self->current ) if $rebuilding_current_branch;
$self->remove_old_branches;
-
if ( $self->is_multi_version ) {
- say " - Writing versions TOC";
- $toc->write($dir);
+ if ( $rebuilding_any_branch ) {
+ printf(" - %40.40s: Writing versions TOC\n", $self->title);
+ $toc->write($dir);
+ }
return {
title => "$title [" . $self->branch_title( $self->current ) . "\\]",
url => $self->prefix . '/current/index.html',
@@ -208,16 +210,27 @@ sub build {
section_title => $self->section_title()
};
}
-
- say " - Writing redirect to current branch";
- write_html_redirect( $dir, "current/index.html" );
-
+ if ( $rebuilding_any_branch ) {
+ printf(" - %40.40s: Writing redirect to current branch...\n", $self->title);
+ write_html_redirect( $dir, "current/index.html" );
+ }
return {
title => $title,
url => $self->prefix . '/current/index.html'
};
}
+#===================================
+# Fork a process to build the book if it needs to be built. Returns 0
+# immediately if the book doesn't have to be built. Forks and then returns 1
+# immediately if the book *does* have to be built. To get the success or
+# failure of the build you must wait on the $pm argument for the children to
+# join the parent process.
+#
+# branch - The branch being built
+# pm - ProcessManager for forking
+# rebuild - if truthy then we rebuild the book regardless of changes.
+# latest - is this the latest branch of the book?
#===================================
sub _build_book {
#===================================
@@ -231,7 +244,7 @@ sub _build_book {
my $subject = $self->subject;
my $lang = $self->lang;
- return
+ return 0
if -e $branch_dir
&& !$rebuild
&& !$template->md5_changed($branch_dir)
@@ -239,8 +252,8 @@ sub _build_book {
my ( $checkout, $edit_urls, $first_path ) = $source->prepare($self->title, $branch);
- $pm->start($branch) and return;
- say " - Branch: $branch - Building...";
+ $pm->start($branch) and return 1;
+ printf(" - %40.40s: Building %s...\n", $self->title, $branch);
eval {
if ( $self->single ) {
$branch_dir->rmtree;
@@ -288,10 +301,13 @@ sub _build_book {
$self->_add_title_to_toc( $branch, $branch_dir );
}
$checkout->rmtree;
- say " - Branch: $branch - Finished";
+ printf(" - %40.40s: Finished %s\n", $self->title, $branch);
1;
} && $pm->finish;
+ # NOTE: This method is about a screen up with $pm->start so it doesn't
+ # return *anything* here. It just dies if there was a failure so we can
+ # pick that up in the parent process.
my $error = $@;
die "\nERROR building "
@@ -332,7 +348,7 @@ sub _copy_branch_to_current {
#===================================
my ( $self, $branch ) = @_;
- say " - Copying $branch to current";
+ printf(" - %40.40s: Copying %s to current\n", $self->title, $branch);
my $branch_dir = $self->dir->subdir($branch);
my $current_dir = $self->dir->subdir('current');
@@ -389,7 +405,7 @@ sub remove_old_branches {
next unless $child->is_dir;
my $version = $child->basename;
next if $branches{$version};
- say " - Deleting old branch: $version";
+ printf(" - %40.40s: Deleting old branch %s\n", $self->title, $version);
$child->rmtree;
}
}