From 72d82a29350a607d9224e8baddb009951b0482ad Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 27 Feb 2019 21:59:42 -0500 Subject: [PATCH 01/20] Keep hash --- build_docs.pl | 10 ++++++---- lib/ES/Repo.pm | 51 ++++++++++++++++++++++---------------------------- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/build_docs.pl b/build_docs.pl index c4377bd1e5735..34d2d3eec57c4 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -53,7 +53,7 @@ BEGIN GetOptions( $Opts, # - 'all', 'push', 'target_repo=s', 'reference=s', 'rebuild', 'no_fetch', # + 'all', 'push', 'target_repo=s', 'reference=s', 'rebuild', 'keep_hash', # 'single', 'pdf', 'doc=s', 'out=s', 'toc', 'chunk=i', 'open', 'skiplinkcheck', 'linkcheckonly', 'staging', 'procs=i', 'user=s', 'lang=s', 'lenient', 'verbose', 'reload_template', 'resource=s@', 'asciidoctor', 'in_standard_docker', @@ -490,6 +490,7 @@ sub init_repos { user => $Opts->{user}, url => $Opts->{target_repo}, reference => $reference_dir, + keep_hash => 0, # intentionally not passing the tracker because we don't want to use it ); delete $child_dirs{ $target_repo->git_dir->absolute }; @@ -524,6 +525,7 @@ sub init_repos { user => $Opts->{user}, url => $url, reference => $reference_dir, + keep_hash => $Opts->{keep_hash}, ); delete $child_dirs{ $repo->git_dir->absolute }; @@ -533,7 +535,7 @@ sub init_repos { else { $pm->start($name) and next; eval { - $repo->update_from_remote() unless $Opts->{no_fetch}; + $repo->update_from_remote(); 1; } or do { # If creds are invalid, explicitly reject them to try to clear the cache @@ -708,7 +710,7 @@ sub check_args { die('--user not compatible with --doc') if $Opts->{user}; die('--reference not compatible with --doc') if $Opts->{reference}; die('--rebuild not compatible with --doc') if $Opts->{rebuild}; - die('--no_fetch not compatible with --doc') if $Opts->{no_fetch}; + die('--keep_hash not compatible with --doc') if $Opts->{keep_hash}; die('--skiplinkcheck not compatible with --doc') if $Opts->{skiplinkcheck}; die('--linkcheckonly not compatible with --doc') if $Opts->{linkcheckonly}; } else { @@ -771,7 +773,7 @@ sub usage { --skiplinkcheck Omit the step that checks for broken links --linkcheckonly Skips the documentation builds. Checks links only. --rebuild Rebuild all branches of every book regardless of what has changed - --no_fetch Skip fetching updates from source repos + --keep_hash Build docs from the same commit hash as last time General Opts: --staging Use the template from the staging website diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index fcac42a898c0d..28430bd8a60df 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -40,6 +40,7 @@ sub new { url => $url, tracker => $args{tracker}, reference_dir => $reference_dir, + keep_hash => $args{keep_hash}, }, $class; if ( $self->tracker ) { # Only track repos that have a tracker. Other repos are for things like @@ -134,10 +135,9 @@ sub has_changed { my $self = shift; my ( $title, $branch, $path, $asciidoctor ) = @_; - my $old - = $self->tracker->sha_for_branch( $self->name, - $self->_tracker_branch(@_) ) - or return 1; + return 1 if $self->keep_hash; + + my $old = $self->_last_commit(@_) or return 1; local $ENV{GIT_DIR} = $self->git_dir; @@ -174,28 +174,6 @@ sub mark_done { } -#=================================== -sub tree { -#=================================== - my $self = shift; - my ( $branch, $path ) = @_; - - local $ENV{GIT_DIR} = $self->git_dir; - - my @files; - eval { - @files = map { Path::Class::file($_) } split /\0/, - run( qw(git ls-tree -r --name-only -z), $branch, '--', $path ); - 1; - } or do { - my $error = $@; - die "Unknown branch <$branch> in repo <" . $self->name . ">" - if $error =~ /Not a valid object name/; - die $@; - }; - return @files; -} - #=================================== sub extract { #=================================== @@ -203,6 +181,11 @@ sub extract { my ( $branch, $path, $dest ) = @_; local $ENV{GIT_DIR} = $self->git_dir; + unless ( $self->update ) { + $branch = $self->_last_commit(@_); + die "--keep_hash can't be performed if for new repos" unless $branch; + } + $dest->mkpath; my $tar = $dest->file('.temp_git_archive.tar'); die "File <$tar> already exists" if -e $tar; @@ -258,9 +241,10 @@ sub dump_recent_commits { my ( $self, $title, $branch, $src_path ) = @_; local $ENV{GIT_DIR} = $self->git_dir; - my $start = $self->tracker->sha_for_branch( $self->name, - $self->_tracker_branch( $title, $branch, $src_path ) ); - my $rev_range = "$start...$branch"; + my $start = $self->_last_commit( $title, $branch, $src_path ); + my $end = $branch; + $end = $start if $self->keep_hash; + my $rev_range = "$start...$end"; my $commits = eval { decode_utf8 run( 'git', 'log', $rev_range, @@ -324,12 +308,21 @@ sub checkout_to { or die "Error checking out repo <$name>: $@"; } +#=================================== +sub _last_commit { +#=================================== + my $self = shift; + my $tracker_branch = $self->_tracker_branch(@_); + return $self->tracker->sha_for_branch($self->name, $tracker_branch); +} + #=================================== sub name { shift->{name} } sub git_dir { shift->{git_dir} } sub url { shift->{url} } sub tracker { shift->{tracker} } sub reference_dir { shift->{reference_dir} } +sub keep_hash { shift->{keep_hash} } #=================================== 1 From ee327bf3bdcfd4b5717be53ec465f083d00bc388 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 09:28:45 -0500 Subject: [PATCH 02/20] WIP --- build_docs | 22 +++++++++--- build_docs.pl | 15 +++++++- integtest/Makefile | 74 +++++++++++++++++++++++++++------------ integtest/small_conf.yaml | 2 +- lib/ES/Book.pm | 2 +- lib/ES/Repo.pm | 49 +++++++++++++++++++++----- lib/ES/Source.pm | 3 +- 7 files changed, 128 insertions(+), 39 deletions(-) diff --git a/build_docs b/build_docs index 65e8fed8a1c4b..0e41a6db79959 100755 --- a/build_docs +++ b/build_docs @@ -29,7 +29,7 @@ from __future__ import print_function import logging from os import environ, getgid, getuid -from os.path import basename, dirname, exists, isdir, join, realpath +from os.path import basename, dirname, exists, expanduser, isdir, join, realpath import re import subprocess from sys import platform, version_info @@ -103,8 +103,7 @@ def run_build_docs(args): '-v', '%s:/doc/%s:ro,cached' % (repo_root, repo_name) ]) - build_docs_args.append( - '/doc/' + repo_name + path.replace(repo_root, '')) + return '/doc/' + repo_name + path.replace(repo_root, '') open_browser = False args = Args(args) @@ -150,7 +149,8 @@ def run_build_docs(args): doc_file = realpath(args.next_arg_or_err()) if not exists(doc_file): raise ArgError("Can't find --doc %s" % doc_file) - mount_docs_repo_and_dockerify_path(dirname(doc_file), doc_file) + build_docs_args.append(mount_docs_repo_and_dockerify_path( + dirname(doc_file), doc_file)) saw_doc = True elif arg == '--open': docker_args.extend(['--publish', '8000:8000/tcp']) @@ -194,7 +194,19 @@ def run_build_docs(args): resource_dir = realpath(args.next_arg_or_err()) if not isdir(resource_dir): raise ArgError("Can't find --resource %s" % resource_dir) - mount_docs_repo_and_dockerify_path(resource_dir, resource_dir) + build_docs_args.append(mount_docs_repo_and_dockerify_path( + resource_dir, resource_dir)) + elif arg == '--sub_dir': + sub = args.next_arg_or_err() + m = re.match('(?P[^:]+):(?P[^:]+):(?P.+)', sub) + if not m: + raise ArgError("Invalid --sub_dir %s" % sub) + sub_dir = realpath(expanduser(m.group('dir'))) + if not exists(sub_dir): + raise ArgError("Can't find --sub_dir %s" % sub_dir) + mounted_path = mount_docs_repo_and_dockerify_path(sub_dir, sub_dir) + build_docs_args.append("%s:%s:%s" % ( + m.group('repo'), m.group('branch'), mounted_path)) arg = args.next_arg() if saw_doc and not saw_out: diff --git a/build_docs.pl b/build_docs.pl index 34d2d3eec57c4..49f1f29f675f7 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -53,7 +53,7 @@ BEGIN GetOptions( $Opts, # - 'all', 'push', 'target_repo=s', 'reference=s', 'rebuild', 'keep_hash', # + 'all', 'push', 'target_repo=s', 'reference=s', 'rebuild', 'keep_hash', 'sub_dir=s@', 'single', 'pdf', 'doc=s', 'out=s', 'toc', 'chunk=i', 'open', 'skiplinkcheck', 'linkcheckonly', 'staging', 'procs=i', 'user=s', 'lang=s', 'lenient', 'verbose', 'reload_template', 'resource=s@', 'asciidoctor', 'in_standard_docker', @@ -550,12 +550,23 @@ sub init_repos { } $pm->wait_all_children; + # Parse the --sub_dir options and attach the to the repo + my %sub_dirs = (); + foreach (@{ $Opts->{sub_dir} }) { + die "invalid --sub_dir $_" + unless /(?[^:]+):(?[^:]+):(?.+)/; + my $dir = dir($+{dir})->absolute; + die "--sub_dir $dir doesn't exist" unless -e $dir; + ES::Repo->get_repo($+{repo})->add_sub_dir($+{branch}, $dir); + } + for ( keys %child_dirs ) { my $dir = dir($_); next unless -d $dir; say "Removing old repo <" . $dir->basename . ">"; $dir->rmtree; } + return ($repos_dir, $temp_dir, $target_repo, $target_repo_checkout); } @@ -774,6 +785,8 @@ sub usage { --linkcheckonly Skips the documentation builds. Checks links only. --rebuild Rebuild all branches of every book regardless of what has changed --keep_hash Build docs from the same commit hash as last time + --sub_dir Use a directory as a branch of some repo + (eg --sub_dir elasticsearch:master:~/Code/elasticsearch) General Opts: --staging Use the template from the staging website diff --git a/integtest/Makefile b/integtest/Makefile index 760dc59401dc6..e517d482bffc8 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -13,7 +13,8 @@ check: \ minimal_expected_files minimal_same_files \ includes_expected_files includes_same_files \ readme_expected_files readme_same_files \ - small_all_expected_files + small_all_expected_files \ + keep_hash_expected_files .PHONY: style style: html_diff @@ -85,27 +86,56 @@ small_all_expected_files: /tmp/small_all grep '' $^/html/test/index.html > /dev/null [ -s $^/html/test/current/index.html ] +define BUILD_MINIMAL= +# First build a repository to use as the source. +rm -rf /tmp/source +git init /tmp/source +cp minimal.asciidoc /tmp/source/index.asciidoc +cd /tmp/source && \ + git add . && \ + git commit -m 'minimal' + +# Initialize a bare repository that the docs build process can use as a +# remote. It is used to pushing to github but it can push to a remote on +# the filesystem just fine. +git init --bare $@.git + +# Actually build the docs +/docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo $@.git \ + --conf small_conf.yaml + +# Check out the files we just built +git clone $@.git $@ +endef + .PRECIOUS: /tmp/small_all /tmp/small_all: # Builds "--all" documentation specified by by the "small_conf.yaml" file. - - # First build a repository to use as the source. - rm -rf /tmp/source - git init /tmp/source - cp minimal.asciidoc /tmp/source/ - cd /tmp/source && \ - git add . && \ - git commit -m 'minimal' - - # Initialize a bare repository that the docs build process can use as a - # remote. It is used to pushing to github but it can push to a remote on - # the filesystem just fine. - git init --bare /tmp/small_all.git - - # Actually build the docs - /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo /tmp/small_all.git \ - --conf small_conf.yaml - - # Check out the files we just built - git clone /tmp/small_all.git /tmp/small_all + $(BUILD_MINIMAL) + + +# .PRECIOUS: /tmp/keep_hash +# /tmp/keep_hash: +# # Builds "--all" documentation specified by by the "small_conf.yaml" file. + +# # First build a repository to use as the source. +# rm -rf /tmp/source +# git init /tmp/source +# cp minimal.asciidoc /tmp/source/index.asciidoc +# cd /tmp/source && \ +# git add . && \ +# git commit -m 'minimal' + +# # Initialize a bare repository that the docs build process can use as a +# # remote. It is used to pushing to github but it can push to a remote on +# # the filesystem just fine. +# git init --bare $^.git + +# # Actually build the docs +# /docs_build/build_docs.pl --in_standard_docker --all --push \ +# --target_repo $^.git \ +# --conf small_conf.yaml + +# # Check out the files we just built +# git clone $^.git $^ diff --git a/integtest/small_conf.yaml b/integtest/small_conf.yaml index 10c1e155dd8f2..0b12875174074 100644 --- a/integtest/small_conf.yaml +++ b/integtest/small_conf.yaml @@ -44,7 +44,7 @@ contents: prefix: test current: master branches: [ master ] - index: minimal.asciidoc + index: index.asciidoc tags: test tag subject: Test sources: diff --git a/lib/ES/Book.pm b/lib/ES/Book.pm index 4c7a35bce2b1d..542c646f8edc2 100644 --- a/lib/ES/Book.pm +++ b/lib/ES/Book.pm @@ -236,7 +236,7 @@ sub _build_book { && !$template->md5_changed($branch_dir) && !$source->has_changed( $self->title, $branch, $self->asciidoctor ); - my ( $checkout, $first_path ) = $source->prepare($branch); + my ( $checkout, $first_path ) = $source->prepare($self->title, $branch); $pm->start($branch) and return; say " - Branch: $branch - Building..."; diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 28430bd8a60df..249ba8648b66a 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -41,6 +41,7 @@ sub new { tracker => $args{tracker}, reference_dir => $reference_dir, keep_hash => $args{keep_hash}, + sub_dirs => {}, }, $class; if ( $self->tracker ) { # Only track repos that have a tracker. Other repos are for things like @@ -129,15 +130,24 @@ sub _reference_args { return (); } +#=================================== +sub add_sub_dir { +#=================================== + my ( $self, $branch, $dir ) = @_; + $self->{sub_dirs}->{$branch} = $dir; + return (); +} + #=================================== sub has_changed { #=================================== my $self = shift; my ( $title, $branch, $path, $asciidoctor ) = @_; - return 1 if $self->keep_hash; + return 1 if exists $self->{sub_dirs}->{$branch}; + return 0 if $self->keep_hash; - my $old = $self->_last_commit(@_) or return 1; + my $old = $self->_last_commit_info(@_) or return 1; local $ENV{GIT_DIR} = $self->git_dir; @@ -171,21 +181,28 @@ sub mark_done { $new .= '|asciidoctor' if $asciidoctor; $self->tracker->set_sha_for_branch( $self->name, $self->_tracker_branch(@_), $new ); - } #=================================== sub extract { #=================================== my $self = shift; - my ( $branch, $path, $dest ) = @_; - local $ENV{GIT_DIR} = $self->git_dir; + my ( $title, $branch, $path, $dest ) = @_; + + if ( exists $self->{sub_dirs}->{$branch} ) { + return if -e $dest; # Already done! + symlink( $self->{sub_dirs}->{$branch}, $dest ); + return; + } - unless ( $self->update ) { + if ( $self->keep_hash ) { $branch = $self->_last_commit(@_); - die "--keep_hash can't be performed if for new repos" unless $branch; + say "ASDFADFADSF " . $self->name . " $branch\n"; + die "--keep_hash can't be performed for new repos" unless $branch; } + local $ENV{GIT_DIR} = $self->git_dir; + $dest->mkpath; my $tar = $dest->file('.temp_git_archive.tar'); die "File <$tar> already exists" if -e $tar; @@ -300,6 +317,7 @@ sub checkout_to { #=================================== my ( $self, $destination ) = @_; + die 'sub_dir not supported with checkout_to' if %{ $self->{sub_dirs}}; my $name = $self->name; eval { run qw(git clone), $self->git_dir, $destination; @@ -308,12 +326,27 @@ sub checkout_to { or die "Error checking out repo <$name>: $@"; } +#=================================== +# Information about the last commit, *not* including flags like `asciidoctor.` #=================================== sub _last_commit { +#=================================== + my $self = shift; + my $sha = $self->_last_commit_info(@_); + $sha =~ s/\|.+$//; # Strip |asciidoctor if it is in the hash + return $sha; +} + +#=================================== +# Information about the last commit, including flags like `asciidoctor.` +#=================================== +sub _last_commit_info { #=================================== my $self = shift; my $tracker_branch = $self->_tracker_branch(@_); - return $self->tracker->sha_for_branch($self->name, $tracker_branch); + my $sha = $self->tracker->sha_for_branch($self->name, $tracker_branch); + $sha =~ s/\|.+$//; # Strip |asciidoctor if it is in the hash + return $sha; } #=================================== diff --git a/lib/ES/Source.pm b/lib/ES/Source.pm index 77e378835d6e1..432210a44383a 100644 --- a/lib/ES/Source.pm +++ b/lib/ES/Source.pm @@ -90,6 +90,7 @@ sub dump_recent_commits { sub prepare { #=================================== my $self = shift; + my $title = shift; my $branch = shift; my %entries; @@ -101,7 +102,7 @@ sub prepare { my $prefix = $source->{prefix}; my $path = $source->{path}; - $repo->extract( $branch, $path, $dest->subdir($prefix) ); + $repo->extract( $title, $branch, $path, $dest->subdir($prefix) ); } return ( $dest, $dest->subdir( $self->first->{prefix} ) ); From e26c457681473fca1db7f4ab8f926ff687af3779 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 09:37:55 -0500 Subject: [PATCH 03/20] WIP --- integtest/Makefile | 107 ++++++++++++++++++++++++--------------------- 1 file changed, 57 insertions(+), 50 deletions(-) diff --git a/integtest/Makefile b/integtest/Makefile index e517d482bffc8..55253c16f9f31 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -81,61 +81,68 @@ endef .PHONY: small_all_expected_files small_all_expected_files: /tmp/small_all + $(MINIMAL_EXPECTED_FILES) + +.PRECIOUS: /tmp/small_all +/tmp/small_all: + $(BUILD_MINIMAL) + +.PHONY: keep_hash_expected_files +keep_hash_expected_files: /tmp/keep_hash + # We expact the same files as the minimal because we won't have made + # any changes + $(MINIMAL_EXPECTED_FILES) + +.PRECIOUS: /tmp/keep_hash +/tmp/keep_hash: + # Test the `--all --keep_hash` combination + + # Build the "minimal" source file + $(BUILD_MINIMAL) + + # REPLACE the minimal documentation in the source repo + cp ../README.asciidoc /tmp/source/index.asciidoc + cd /tmp/source && \ + git add . && \ + git commit -m 'README' + + # Rebuild the docs with --keep_hash which should ignore the replacement + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --keep_hash \ + --target_repo $@.git \ + --conf small_conf.yaml + +define MINIMAL_EXPECTED_FILES= + # Checks that $^ contains the expected result of building the "minimal" + # source file [ -s $^/html/branches.yaml ] grep 'Test book' $^/html/index.html > /dev/null grep '' $^/html/test/index.html > /dev/null [ -s $^/html/test/current/index.html ] +endef define BUILD_MINIMAL= -# First build a repository to use as the source. -rm -rf /tmp/source -git init /tmp/source -cp minimal.asciidoc /tmp/source/index.asciidoc -cd /tmp/source && \ - git add . && \ - git commit -m 'minimal' - -# Initialize a bare repository that the docs build process can use as a -# remote. It is used to pushing to github but it can push to a remote on -# the filesystem just fine. -git init --bare $@.git - -# Actually build the docs -/docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo $@.git \ - --conf small_conf.yaml - -# Check out the files we just built -git clone $@.git $@ + # Builds `--all` docs using a "minimal" source file + + # First build a repository to use as the source. + rm -rf /tmp/source + git init /tmp/source + cp minimal.asciidoc /tmp/source/index.asciidoc + cd /tmp/source && \ + git add . && \ + git commit -m 'minimal' + + # Initialize a bare repository that the docs build process can use as a + # remote. It is used to pushing to github but it can push to a remote on + # the filesystem just fine. + git init --bare $@.git + + # Actually build the docs + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo $@.git \ + --conf small_conf.yaml + + # Check out the files we just built + git clone $@.git $@ endef -.PRECIOUS: /tmp/small_all -/tmp/small_all: - # Builds "--all" documentation specified by by the "small_conf.yaml" file. - $(BUILD_MINIMAL) - - -# .PRECIOUS: /tmp/keep_hash -# /tmp/keep_hash: -# # Builds "--all" documentation specified by by the "small_conf.yaml" file. - -# # First build a repository to use as the source. -# rm -rf /tmp/source -# git init /tmp/source -# cp minimal.asciidoc /tmp/source/index.asciidoc -# cd /tmp/source && \ -# git add . && \ -# git commit -m 'minimal' - -# # Initialize a bare repository that the docs build process can use as a -# # remote. It is used to pushing to github but it can push to a remote on -# # the filesystem just fine. -# git init --bare $^.git - -# # Actually build the docs -# /docs_build/build_docs.pl --in_standard_docker --all --push \ -# --target_repo $^.git \ -# --conf small_conf.yaml - -# # Check out the files we just built -# git clone $^.git $^ From b1acdaa2ca3c0f02b405de692025943114c7d9ed Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 11:21:10 -0500 Subject: [PATCH 04/20] WIP --- build_docs.pl | 1 + integtest/Makefile | 98 +++++++++++++++++++++++++++++----------------- 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/build_docs.pl b/build_docs.pl index 52a8be6323d5e..7ba55f16f8e9b 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -721,6 +721,7 @@ sub check_args { die('--reference not compatible with --doc') if $Opts->{reference}; die('--rebuild not compatible with --doc') if $Opts->{rebuild}; die('--keep_hash not compatible with --doc') if $Opts->{keep_hash}; + die('--sub_dir not compatible with --doc') if $Opts->{sub_dir}; die('--skiplinkcheck not compatible with --doc') if $Opts->{skiplinkcheck}; die('--linkcheckonly not compatible with --doc') if $Opts->{linkcheckonly}; } else { diff --git a/integtest/Makefile b/integtest/Makefile index 55253c16f9f31..9ef8135b6324d 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -13,8 +13,9 @@ check: \ minimal_expected_files minimal_same_files \ includes_expected_files includes_same_files \ readme_expected_files readme_same_files \ - small_all_expected_files \ - keep_hash_expected_files + simple_all_expected_files \ + keep_hash_expected_files \ + sub_dir_expected_files .PHONY: style style: html_diff @@ -79,26 +80,16 @@ endef /tmp/%_asciidoctor: $(BD) --asciidoctor --doc $*.asciidoc -.PHONY: small_all_expected_files -small_all_expected_files: /tmp/small_all - $(MINIMAL_EXPECTED_FILES) - -.PRECIOUS: /tmp/small_all -/tmp/small_all: - $(BUILD_MINIMAL) +.PHONY: simple_all_expected_files +simple_all_expected_files: + # Test the simplest possible `--all` invocation + $(call BUILD_MINIMAL_ALL,/tmp/small_all) + $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/small_all) .PHONY: keep_hash_expected_files -keep_hash_expected_files: /tmp/keep_hash - # We expact the same files as the minimal because we won't have made - # any changes - $(MINIMAL_EXPECTED_FILES) - -.PRECIOUS: /tmp/keep_hash -/tmp/keep_hash: - # Test the `--all --keep_hash` combination - - # Build the "minimal" source file - $(BUILD_MINIMAL) +keep_hash_expected_files: + # Test that `--all --keep_hash` doesn't pull new updates + $(call BUILD_MINIMAL_ALL,/tmp/keep_hash) # REPLACE the minimal documentation in the source repo cp ../README.asciidoc /tmp/source/index.asciidoc @@ -109,19 +100,50 @@ keep_hash_expected_files: /tmp/keep_hash # Rebuild the docs with --keep_hash which should ignore the replacement /docs_build/build_docs.pl --in_standard_docker --all --push \ --keep_hash \ - --target_repo $@.git \ - --conf small_conf.yaml + --target_repo /tmp/keep_hash.git \ + --conf small_conf.yaml | tee /tmp/out + $(call GREP,'No changes to push',/tmp/out) + + # We expact the same files as the minimal because we the changes that we + # make shouldn't be included + $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/keep_hash) + +.PHONY: sub_dir_expected_file +sub_dir_expected_files: + # Test that `--all --sub_dir` substitutes a directory for a branch of + # a repo. + + # We still need to build the source repo because the script wants to fetch + # it just in case we need another branch. + rm -rf /tmp/source + git init /tmp/source + cd /tmp/source && git commit --allow-empty -m "empty" + + # Setup the directory we'd like to substitute + rm -rf /tmp/to_sub + mkdir /tmp/to_sub + cp minimal.asciidoc /tmp/to_sub/index.asciidoc + + git init --bare /tmp/sub_dir.git + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo /tmp/sub_dir.git \ + --conf small_conf.yaml \ + --sub_dir source:master:/tmp/to_sub -define MINIMAL_EXPECTED_FILES= - # Checks that $^ contains the expected result of building the "minimal" - # source file - [ -s $^/html/branches.yaml ] - grep 'Test book' $^/html/index.html > /dev/null - grep '' $^/html/test/index.html > /dev/null - [ -s $^/html/test/current/index.html ] + $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/sub_dir) + + +define GREP= + # grep for a string in a file, outputting the whole file if there isn't + # a match. + grep $(1) $(2) > /dev/null || { \ + echo "Couldn't find $(1) in $(2):"; \ + cat $(2); \ + false; \ + } endef -define BUILD_MINIMAL= +define BUILD_MINIMAL_ALL= # Builds `--all` docs using a "minimal" source file # First build a repository to use as the source. @@ -135,14 +157,20 @@ define BUILD_MINIMAL= # Initialize a bare repository that the docs build process can use as a # remote. It is used to pushing to github but it can push to a remote on # the filesystem just fine. - git init --bare $@.git + git init --bare $(1).git # Actually build the docs /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo $@.git \ + --target_repo $(1).git \ --conf small_conf.yaml - - # Check out the files we just built - git clone $@.git $@ endef +define MINIMAL_ALL_EXPECTED_FILES= + # Checks that $(1).git contains the expected result of building the + # "minimal" source file + git clone $(1).git $(1) + [ -s $(1)/html/branches.yaml ] + $(call GREP,'Test book',$(1)/html/index.html) + $(call GREP,'',$(1)/html/test/index.html) + [ -s $(1)/html/test/current/index.html ] +endef From 26a0d97a62d64469ea6de20afda51b551133b7ca Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 12:34:58 -0500 Subject: [PATCH 05/20] More tests --- integtest/Makefile | 74 ++++++++++++++++++++++++----- integtest/includes_source2.asciidoc | 9 ++++ integtest/two_repos_conf.yaml | 46 ++++++++++++++++++ lib/ES/Repo.pm | 13 +++-- 4 files changed, 127 insertions(+), 15 deletions(-) create mode 100644 integtest/includes_source2.asciidoc create mode 100644 integtest/two_repos_conf.yaml diff --git a/integtest/Makefile b/integtest/Makefile index 9ef8135b6324d..26cc296b8f4c2 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -15,7 +15,8 @@ check: \ readme_expected_files readme_same_files \ simple_all_expected_files \ keep_hash_expected_files \ - sub_dir_expected_files + sub_dir_expected_files \ + keep_hash_and_sub_dir_expected_files .PHONY: style style: html_diff @@ -83,8 +84,8 @@ endef .PHONY: simple_all_expected_files simple_all_expected_files: # Test the simplest possible `--all` invocation - $(call BUILD_MINIMAL_ALL,/tmp/small_all) - $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/small_all) + $(call BUILD_MINIMAL_ALL,/tmp/simple_all) + $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/simple_all) .PHONY: keep_hash_expected_files keep_hash_expected_files: @@ -99,9 +100,9 @@ keep_hash_expected_files: # Rebuild the docs with --keep_hash which should ignore the replacement /docs_build/build_docs.pl --in_standard_docker --all --push \ - --keep_hash \ --target_repo /tmp/keep_hash.git \ - --conf small_conf.yaml | tee /tmp/out + --conf small_conf.yaml \ + --keep_hash | tee /tmp/out $(call GREP,'No changes to push',/tmp/out) # We expact the same files as the minimal because we the changes that we @@ -132,10 +133,55 @@ sub_dir_expected_files: $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/sub_dir) +keep_hash_and_sub_dir_expected_files: + # Test that `--all --keep_hash --sub_dir` keeps hashes the same for repos + # not specified by --sub_dir but forces rebuilding all books that include + # --sub_dir. + + $(call INIT_REPO_WITH_FILE,/tmp/source1,includes_source2.asciidoc,index.asciidoc) + $(call INIT_REPO_WITH_FILE,/tmp/source2,included.asciidoc,index.asciidoc) + git init --bare /tmp/keep_hash_and_sub_dir.git + + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo /tmp/keep_hash_and_sub_dir.git \ + --conf two_repos_conf.yaml + + # Move a "bad" file into source2 so we can be sure we're not picking it up + cp ../README.asciidoc /tmp/source2/index.asciidoc + cd /tmp/source2 && \ + git add . && \ + git commit -m 'README' + + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo /tmp/keep_hash_and_sub_dir.git \ + --conf two_repos_conf.yaml \ + --keep_hash | tee /tmp/out + $(call GREP,'No changes to push',/tmp/out) + + # Setup the directory we'd like to substitute + rm -rf /tmp/to_sub + mkdir /tmp/to_sub + cp includes_source2.asciidoc /tmp/to_sub/index.asciidoc + echo "extra extra extra" >> /tmp/to_sub/index.asciidoc + + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo /tmp/keep_hash_and_sub_dir.git \ + --conf two_repos_conf.yaml \ + --keep_hash --sub_dir source1:master:/tmp/to_sub | tee /tmp/out + $(call GREP,'Pushing changes',/tmp/out) + + git clone /tmp/keep_hash_and_sub_dir.git /tmp/keep_hash_and_sub_dir + $(call GREP,'extra extra extra',/tmp/keep_hash_and_sub_dir/html/test/current/_chapter.html) + define GREP= # grep for a string in a file, outputting the whole file if there isn't # a match. + [ -e $(2) ] || { \ + echo "can't find $(2)"; \ + ls $$(dirname $(2)); \ + false; \ + } grep $(1) $(2) > /dev/null || { \ echo "Couldn't find $(1) in $(2):"; \ cat $(2); \ @@ -147,12 +193,7 @@ define BUILD_MINIMAL_ALL= # Builds `--all` docs using a "minimal" source file # First build a repository to use as the source. - rm -rf /tmp/source - git init /tmp/source - cp minimal.asciidoc /tmp/source/index.asciidoc - cd /tmp/source && \ - git add . && \ - git commit -m 'minimal' + $(call INIT_REPO_WITH_FILE,/tmp/source,minimal.asciidoc,index.asciidoc) # Initialize a bare repository that the docs build process can use as a # remote. It is used to pushing to github but it can push to a remote on @@ -174,3 +215,14 @@ define MINIMAL_ALL_EXPECTED_FILES= $(call GREP,'',$(1)/html/test/index.html) [ -s $(1)/html/test/current/index.html ] endef + +define INIT_REPO_WITH_FILE= + # Initializes the repo at $(1) and commits the file in $(2) with the + # name $(3) + rm -rf $(1) + git init $(1) + cp $(2) $(1)/$(3) + cd $(1) && \ + git add . && \ + git commit -m 'init' +endef \ No newline at end of file diff --git a/integtest/includes_source2.asciidoc b/integtest/includes_source2.asciidoc new file mode 100644 index 0000000000000..468a81eda96af --- /dev/null +++ b/integtest/includes_source2.asciidoc @@ -0,0 +1,9 @@ += Title + +== Chapter + +I include simple between here + +include::../source2/index.asciidoc[] + +and here. diff --git a/integtest/two_repos_conf.yaml b/integtest/two_repos_conf.yaml new file mode 100644 index 0000000000000..4f6aba9086f60 --- /dev/null +++ b/integtest/two_repos_conf.yaml @@ -0,0 +1,46 @@ +# This conf file builds a book using /tmp/source1 and /tmp/source2. See +# small_conf.yaml for description of the sections. + +template: + path: .template/ + branch: + default: + base_url: 'https://www.elastic.co/' + template_url: 'https://www.elastic.co/guide_template' + staging: + base_url: 'https://stag-www.elastic.co/' + template_url: 'https://stag-www.elastic.co/guide_template' + defaults: + POSTHEAD: | + + FINAL: | + + + +paths: + build: html/ + branch_tracker: html/branches.yaml + repos: /tmp/repos/ + +repos: + source1: /tmp/source1 + source2: /tmp/source2 + +contents_title: Elastic Stack and Product Documentation + +contents: + - + title: Test book + prefix: test + current: master + branches: [ master ] + index: index.asciidoc + tags: test tag + subject: Test + sources: + - + repo: source1 + path: . + - + repo: source2 + path: . diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 249ba8648b66a..5516acfe2f6b9 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -197,7 +197,6 @@ sub extract { if ( $self->keep_hash ) { $branch = $self->_last_commit(@_); - say "ASDFADFADSF " . $self->name . " $branch\n"; die "--keep_hash can't be performed for new repos" unless $branch; } @@ -256,10 +255,17 @@ sub edit_url_for_url_and_branch { sub dump_recent_commits { #=================================== my ( $self, $title, $branch, $src_path ) = @_; - local $ENV{GIT_DIR} = $self->git_dir; + my $description = $self->name . "/$title:$branch:$src_path"; + if ( exists $self->{sub_dirs}->{$branch} ) { + return "Used " . $self->{sub_dirs}->{$branch} . + " for $description\n"; + } + + local $ENV{GIT_DIR} = $self->git_dir; my $start = $self->_last_commit( $title, $branch, $src_path ); my $end = $branch; + say "ASDASDFSDSAF" . $self->name . "\n" if $self->keep_hash; $end = $start if $self->keep_hash; my $rev_range = "$start...$end"; @@ -276,8 +282,7 @@ sub dump_recent_commits { '-n', 10, '--abbrev-commit', '--date=relative', '--', $src_path ); } - my $header - = "Recent commits in " . $self->name . "/$title:$branch:$src_path:"; + my $header = "Recent commits in $description"; return $header . "\n" . ( '-' x length($header) ) . "\n" From 78af0ffe542cb99474ba234beb3ecdc9da053ccf Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 12:52:58 -0500 Subject: [PATCH 06/20] WIP --- build_docs.pl | 4 +++- lib/ES/Repo.pm | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build_docs.pl b/build_docs.pl index 7ba55f16f8e9b..a1766b0218287 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -462,7 +462,9 @@ sub init_repos { my %child_dirs = map { $_ => 1 } $repos_dir->children; - my $temp_dir = $running_in_standard_docker ? dir('/tmp/docsbuild') : $repos_dir->subdir('.temp'); + # NOCOMMIT + # my $temp_dir = $running_in_standard_docker ? dir('/tmp/docsbuild') : $repos_dir->subdir('.temp'); + my $temp_dir = $repos_dir->subdir('.temp'); $temp_dir->rmtree; $temp_dir->mkpath; delete $child_dirs{ $temp_dir->absolute }; diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 5516acfe2f6b9..40e7405a5b5e1 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -191,6 +191,7 @@ sub extract { if ( exists $self->{sub_dirs}->{$branch} ) { return if -e $dest; # Already done! + say "ASDF symlinking $title $path $dest\n"; symlink( $self->{sub_dirs}->{$branch}, $dest ); return; } @@ -202,6 +203,7 @@ sub extract { local $ENV{GIT_DIR} = $self->git_dir; + say "ASDF extracting $title $path $dest\n"; $dest->mkpath; my $tar = $dest->file('.temp_git_archive.tar'); die "File <$tar> already exists" if -e $tar; @@ -265,7 +267,6 @@ sub dump_recent_commits { local $ENV{GIT_DIR} = $self->git_dir; my $start = $self->_last_commit( $title, $branch, $src_path ); my $end = $branch; - say "ASDASDFSDSAF" . $self->name . "\n" if $self->keep_hash; $end = $start if $self->keep_hash; my $rev_range = "$start...$end"; From 61718efab2904e5ddf9360215cd4460580daf2c4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 15:10:27 -0500 Subject: [PATCH 07/20] Allow building all docs with a local repo We current offer two ways to build the docs: 1. Build an arbitrary book with `--doc` and its companion arguments like `--resource` and `--asciidoctor`. 2. Build *all* of the books that Elastic publishes with `--all`. Building an arbitrary book is nice and quick because it is have very little overhead on top of the time it takes to build the actual book. But it isn't always the best test because the command that you use to build the book might not exactly match the invocation that `--all` uses to build the book. Furthermore, building a single book is not usually enough! When you are testing documentation changes that you've made locally you often don't know all of the books that your changes might effect. The Elasticsearch repository is used in about a dozen books in combination with a hand full of other repositories. Building all of the docs is quite slow and you might run into errors that you didn't cause. It is exactly what we need to build the docs for the public web site but it isn't right for testing some changes that you make locally. It also doesn't line up properly with local development as it only knows how to pull repos from the Elastic organization at github. This change adds a "third way" to build test the docs that splits the difference between the two options. It uses the same mechanisms that we usually use to build all of the docs but allows you to substitute local directories in place of some branch of some repo: ``` ./build_docs --all --target_repo git@github.com:elastic/built-docs.git \ --keep_hash \ --sub_dir elasticsearch:master:~/Code/elasticsearch ``` This is nice because it: 1. Rebuilds all of the books that your local directory is involved with 2. Uses standard switches when building each of those books 3. Does not rebuild books that don't use your local directory That third thing is actually caused by that new `--keep_hash` flag. It causes the build's up-to-date checking to use the same hash that was used the last time the book was built. `--keep_hash` has uses beyond being paired with `--sub_dir`. In particular, it is useful when testing what happens when you switch a book to Asciidoctor. You can switch the book to Asciidoctor in `conf.yaml` and rebuild with `--keep_hash` and the build will *only* have the asciidoctor change in it. This also drop `--no_fetch` because `--keep_hash` is better in almost every way. The goal of `--sub_dir` isn't so much that people will execute it locally, but that we can execute it on CI. I'm sure folks will execute it locally but it pretty heavy because, just like a normal `--all` build it has to: 1. Clone the `built-docs` repo or fetch updates if it is already cloned. 2. Check out the `built-docs` repo. This is truly slow because it has many, many files in it. 3. Clone all of the repos used to build every book or fetch updates if they are already cloned. I suspect in time we can fix some of these. But for now, this is a heavy thing. --- build_docs.pl | 7 +++--- integtest/Makefile | 12 ++++++--- integtest/includes_source2.asciidoc | 2 +- integtest/two_repos_conf.yaml | 4 +-- lib/ES/Book.pm | 3 ++- lib/ES/Repo.pm | 38 +++++++++++++++++++++-------- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/build_docs.pl b/build_docs.pl index a1766b0218287..2cefcc3569b47 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -283,7 +283,8 @@ sub build_all { } push_changes($build_dir, $target_repo, $target_repo_checkout) if $Opts->{push}; - $temp_dir->rmtree; + # NOCOMMIT + # $temp_dir->rmtree; } #=================================== @@ -462,9 +463,7 @@ sub init_repos { my %child_dirs = map { $_ => 1 } $repos_dir->children; - # NOCOMMIT - # my $temp_dir = $running_in_standard_docker ? dir('/tmp/docsbuild') : $repos_dir->subdir('.temp'); - my $temp_dir = $repos_dir->subdir('.temp'); + my $temp_dir = $running_in_standard_docker ? dir('/tmp/docsbuild') : $repos_dir->subdir('.temp'); $temp_dir->rmtree; $temp_dir->mkpath; delete $child_dirs{ $temp_dir->absolute }; diff --git a/integtest/Makefile b/integtest/Makefile index 26cc296b8f4c2..d5677d515d290 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -125,6 +125,7 @@ sub_dir_expected_files: mkdir /tmp/to_sub cp minimal.asciidoc /tmp/to_sub/index.asciidoc + rm -rf /tmp/sub_dir.git git init --bare /tmp/sub_dir.git /docs_build/build_docs.pl --in_standard_docker --all --push \ --target_repo /tmp/sub_dir.git \ @@ -138,8 +139,9 @@ keep_hash_and_sub_dir_expected_files: # not specified by --sub_dir but forces rebuilding all books that include # --sub_dir. - $(call INIT_REPO_WITH_FILE,/tmp/source1,includes_source2.asciidoc,index.asciidoc) + $(call INIT_REPO_WITH_FILE,/tmp/source1,includes_source2.asciidoc,docs/index.asciidoc) $(call INIT_REPO_WITH_FILE,/tmp/source2,included.asciidoc,index.asciidoc) + rm -rf /tmp/keep_hash_and_sub_dir.git git init --bare /tmp/keep_hash_and_sub_dir.git /docs_build/build_docs.pl --in_standard_docker --all --push \ @@ -160,9 +162,9 @@ keep_hash_and_sub_dir_expected_files: # Setup the directory we'd like to substitute rm -rf /tmp/to_sub - mkdir /tmp/to_sub - cp includes_source2.asciidoc /tmp/to_sub/index.asciidoc - echo "extra extra extra" >> /tmp/to_sub/index.asciidoc + mkdir -p /tmp/to_sub/docs + cp includes_source2.asciidoc /tmp/to_sub/docs/index.asciidoc + echo "extra extra extra" >> /tmp/to_sub/docs/index.asciidoc /docs_build/build_docs.pl --in_standard_docker --all --push \ --target_repo /tmp/keep_hash_and_sub_dir.git \ @@ -198,6 +200,7 @@ define BUILD_MINIMAL_ALL= # Initialize a bare repository that the docs build process can use as a # remote. It is used to pushing to github but it can push to a remote on # the filesystem just fine. + rm -rf $(1).git git init --bare $(1).git # Actually build the docs @@ -221,6 +224,7 @@ define INIT_REPO_WITH_FILE= # name $(3) rm -rf $(1) git init $(1) + mkdir -p $$(dirname $(1)/$(3)) cp $(2) $(1)/$(3) cd $(1) && \ git add . && \ diff --git a/integtest/includes_source2.asciidoc b/integtest/includes_source2.asciidoc index 468a81eda96af..9f7b8e73e913b 100644 --- a/integtest/includes_source2.asciidoc +++ b/integtest/includes_source2.asciidoc @@ -4,6 +4,6 @@ I include simple between here -include::../source2/index.asciidoc[] +include::../../source2/index.asciidoc[] and here. diff --git a/integtest/two_repos_conf.yaml b/integtest/two_repos_conf.yaml index 4f6aba9086f60..ae1c77e14f473 100644 --- a/integtest/two_repos_conf.yaml +++ b/integtest/two_repos_conf.yaml @@ -34,13 +34,13 @@ contents: prefix: test current: master branches: [ master ] - index: index.asciidoc + index: docs/index.asciidoc tags: test tag subject: Test sources: - repo: source1 - path: . + path: docs - repo: source2 path: . diff --git a/lib/ES/Book.pm b/lib/ES/Book.pm index 542c646f8edc2..eec11e0f9600c 100644 --- a/lib/ES/Book.pm +++ b/lib/ES/Book.pm @@ -284,7 +284,8 @@ sub _build_book { ); $self->_add_title_to_toc( $branch, $branch_dir ); } - $checkout->rmtree; + # NOCOMMIT + # $checkout->rmtree; say " - Branch: $branch - Finished"; 1; diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 40e7405a5b5e1..de1af7649c8e0 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -175,10 +175,20 @@ sub mark_done { my $self = shift; my ( $title, $branch, $path, $asciidoctor ) = @_; - local $ENV{GIT_DIR} = $self->git_dir; + if ( exists $self->{sub_dirs}->{$branch} ) { + say "Marking done with local"; + $self->tracker->set_sha_for_branch( $self->name, + $self->_tracker_branch(@_), 'local' ); + return; + } + return if $self->keep_hash; + + local $ENV{GIT_DIR} = $self->git_dir; my $new = sha_for($branch); $new .= '|asciidoctor' if $asciidoctor; + say "Marking done with $new"; + $self->tracker->set_sha_for_branch( $self->name, $self->_tracker_branch(@_), $new ); } @@ -190,9 +200,14 @@ sub extract { my ( $title, $branch, $path, $dest ) = @_; if ( exists $self->{sub_dirs}->{$branch} ) { - return if -e $dest; # Already done! - say "ASDF symlinking $title $path $dest\n"; - symlink( $self->{sub_dirs}->{$branch}, $dest ); + my $realpath = $self->{sub_dirs}->{$branch}->subdir($path); + my $realdest = $dest->subdir($path)->parent; + die "Can't find $realpath" unless -e $realpath; + $realdest->mkpath; + eval { + run qw(cp -r), $realpath, $realdest; + 1; + } or die "Error copying from $realpath: $@"; return; } @@ -203,7 +218,6 @@ sub extract { local $ENV{GIT_DIR} = $self->git_dir; - say "ASDF extracting $title $path $dest\n"; $dest->mkpath; my $tar = $dest->file('.temp_git_archive.tar'); die "File <$tar> already exists" if -e $tar; @@ -266,9 +280,7 @@ sub dump_recent_commits { local $ENV{GIT_DIR} = $self->git_dir; my $start = $self->_last_commit( $title, $branch, $src_path ); - my $end = $branch; - $end = $start if $self->keep_hash; - my $rev_range = "$start...$end"; + my $rev_range = $self->keep_hash ? $start : "$start...$branch"; my $commits = eval { decode_utf8 run( 'git', 'log', $rev_range, @@ -307,8 +319,14 @@ sub all_repo_branches { for my $branch ( sort keys %$shas ) { my $sha = $shas->{$branch}; $sha =~ s/\|.+$//; # Strip |asciidoctor if it is in the hash - my $log = run( qw(git log --oneline -1), $sha ); - my ($msg) = ( $log =~ /^\w+\s+([^\n]+)/ ); + say "Getting log for $sha"; + my $msg; + if ( $sha eq 'local' ) { + $msg = 'local changes'; + } else { + my $log = run( qw(git log --oneline -1), $sha ); + ($msg) = ( $log =~ /^\w+\s+([^\n]+)/ ); + } push @out, sprintf " %-35s %s %s", $branch, substr( $shas->{$branch}, 0, 8 ), $msg; } From 84736eb2e9eb62b0f5ac54e7c00bfce8ea599578 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 15:41:38 -0500 Subject: [PATCH 08/20] Fixups --- conf.yaml | 1 + lib/ES/Repo.pm | 17 +++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/conf.yaml b/conf.yaml index 86894ed806182..5ad97b28bbc12 100644 --- a/conf.yaml +++ b/conf.yaml @@ -339,6 +339,7 @@ contents: index: docs/index.asciidoc tags: Clients/PHP subject: Clients + asciidoctor: false sources: - repo: elasticsearch-php diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index de1af7649c8e0..8eb954cad6b7b 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -175,19 +175,17 @@ sub mark_done { my $self = shift; my ( $title, $branch, $path, $asciidoctor ) = @_; + my $new; if ( exists $self->{sub_dirs}->{$branch} ) { - say "Marking done with local"; - $self->tracker->set_sha_for_branch( $self->name, - $self->_tracker_branch(@_), 'local' ); + $new = 'local'; return; + } elsif ( $self->keep_hash ) { + $new = $self->_last_commit($title, $branch, $path); + } else { + local $ENV{GIT_DIR} = $self->git_dir; + $new = sha_for($branch); } - - return if $self->keep_hash; - - local $ENV{GIT_DIR} = $self->git_dir; - my $new = sha_for($branch); $new .= '|asciidoctor' if $asciidoctor; - say "Marking done with $new"; $self->tracker->set_sha_for_branch( $self->name, $self->_tracker_branch(@_), $new ); @@ -319,7 +317,6 @@ sub all_repo_branches { for my $branch ( sort keys %$shas ) { my $sha = $shas->{$branch}; $sha =~ s/\|.+$//; # Strip |asciidoctor if it is in the hash - say "Getting log for $sha"; my $msg; if ( $sha eq 'local' ) { $msg = 'local changes'; From afc156e87bdf78475f6f818f4e3ebfdf49cf4d0e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 16:38:30 -0500 Subject: [PATCH 09/20] WIP --- conf.yaml | 1 - integtest/Makefile | 2 +- lib/ES/Repo.pm | 16 +++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/conf.yaml b/conf.yaml index 5ad97b28bbc12..86894ed806182 100644 --- a/conf.yaml +++ b/conf.yaml @@ -339,7 +339,6 @@ contents: index: docs/index.asciidoc tags: Clients/PHP subject: Clients - asciidoctor: false sources: - repo: elasticsearch-php diff --git a/integtest/Makefile b/integtest/Makefile index d5677d515d290..db6e7c019d893 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -229,4 +229,4 @@ define INIT_REPO_WITH_FILE= cd $(1) && \ git add . && \ git commit -m 'init' -endef \ No newline at end of file +endef diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 8eb954cad6b7b..7516610e3818b 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -145,18 +145,21 @@ sub has_changed { my ( $title, $branch, $path, $asciidoctor ) = @_; return 1 if exists $self->{sub_dirs}->{$branch}; - return 0 if $self->keep_hash; my $old = $self->_last_commit_info(@_) or return 1; local $ENV{GIT_DIR} = $self->git_dir; - - my $new = sha_for($branch) - or die "Remote branch doesn't exist " - . "in repo " - . $self->name; + my $new; + if ( $self->keep_hash ) { + $new = $self->_last_commit(@_); + } else { + $new = sha_for($branch) or die( + "Remote branch doesn't exist in repo " + . $self->name); + } $new .= '|asciidoctor' if $asciidoctor; + return $old ne $new if $self->keep_hash; return if $old eq $new; my $changed; @@ -366,7 +369,6 @@ sub _last_commit_info { my $self = shift; my $tracker_branch = $self->_tracker_branch(@_); my $sha = $self->tracker->sha_for_branch($self->name, $tracker_branch); - $sha =~ s/\|.+$//; # Strip |asciidoctor if it is in the hash return $sha; } From df51f1d11381a2998217f5e016645a32a3d883e0 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 16:42:27 -0500 Subject: [PATCH 10/20] Fixup fail.... --- integtest/small_conf.yaml | 2 +- integtest/two_repos_conf.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integtest/small_conf.yaml b/integtest/small_conf.yaml index 0b12875174074..36f50aad072e5 100644 --- a/integtest/small_conf.yaml +++ b/integtest/small_conf.yaml @@ -50,4 +50,4 @@ contents: sources: - repo: source - path: . + path: index.asciidoc diff --git a/integtest/two_repos_conf.yaml b/integtest/two_repos_conf.yaml index ae1c77e14f473..9036f7dfe37b5 100644 --- a/integtest/two_repos_conf.yaml +++ b/integtest/two_repos_conf.yaml @@ -43,4 +43,4 @@ contents: path: docs - repo: source2 - path: . + path: index.asciidoc From 71a0e0246934103dc2009cf0a107115fa2afe113 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 16:44:18 -0500 Subject: [PATCH 11/20] pep8 --- build_docs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_docs b/build_docs index 0e41a6db79959..c7328cab5c9d4 100755 --- a/build_docs +++ b/build_docs @@ -29,7 +29,8 @@ from __future__ import print_function import logging from os import environ, getgid, getuid -from os.path import basename, dirname, exists, expanduser, isdir, join, realpath +from os.path import basename, dirname, exists, expanduser, isdir +from os.path import join, realpath import re import subprocess from sys import platform, version_info From 412d80ac6e9e5faa614ccf54d993a1d7bf20927e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 16:47:58 -0500 Subject: [PATCH 12/20] NOCOMMIT --- build_docs.pl | 3 +-- lib/ES/Book.pm | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/build_docs.pl b/build_docs.pl index 2cefcc3569b47..7ba55f16f8e9b 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -283,8 +283,7 @@ sub build_all { } push_changes($build_dir, $target_repo, $target_repo_checkout) if $Opts->{push}; - # NOCOMMIT - # $temp_dir->rmtree; + $temp_dir->rmtree; } #=================================== diff --git a/lib/ES/Book.pm b/lib/ES/Book.pm index eec11e0f9600c..542c646f8edc2 100644 --- a/lib/ES/Book.pm +++ b/lib/ES/Book.pm @@ -284,8 +284,7 @@ sub _build_book { ); $self->_add_title_to_toc( $branch, $branch_dir ); } - # NOCOMMIT - # $checkout->rmtree; + $checkout->rmtree; say " - Branch: $branch - Finished"; 1; From 4cec727ddebf0c5fad270caed4011a2687275bbc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 16:48:58 -0500 Subject: [PATCH 13/20] Fix: --- build_docs.pl | 1 - 1 file changed, 1 deletion(-) diff --git a/build_docs.pl b/build_docs.pl index 7ba55f16f8e9b..c88adc7f0db55 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -565,7 +565,6 @@ sub init_repos { say "Removing old repo <" . $dir->basename . ">"; $dir->rmtree; } - return ($repos_dir, $temp_dir, $target_repo, $target_repo_checkout); } From e22093bfa75b9ce6687c9fdf848d58c74b945b3f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 16:50:11 -0500 Subject: [PATCH 14/20] Un-move --- lib/ES/Repo.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 7516610e3818b..55fb23b181c91 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -146,9 +146,9 @@ sub has_changed { return 1 if exists $self->{sub_dirs}->{$branch}; + local $ENV{GIT_DIR} = $self->git_dir; my $old = $self->_last_commit_info(@_) or return 1; - local $ENV{GIT_DIR} = $self->git_dir; my $new; if ( $self->keep_hash ) { $new = $self->_last_commit(@_); From 36e07c5f84741a8ed6c53bb80764ca0e26c0dc18 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 17:01:44 -0500 Subject: [PATCH 15/20] Little cleaner --- lib/ES/Repo.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index 55fb23b181c91..c302afe698b74 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -181,7 +181,6 @@ sub mark_done { my $new; if ( exists $self->{sub_dirs}->{$branch} ) { $new = 'local'; - return; } elsif ( $self->keep_hash ) { $new = $self->_last_commit($title, $branch, $path); } else { @@ -215,6 +214,7 @@ sub extract { if ( $self->keep_hash ) { $branch = $self->_last_commit(@_); die "--keep_hash can't be performed for new repos" unless $branch; + die "--keep_hash can't build on top of --sub_dir" if $branch eq 'local'; } local $ENV{GIT_DIR} = $self->git_dir; @@ -325,7 +325,7 @@ sub all_repo_branches { $msg = 'local changes'; } else { my $log = run( qw(git log --oneline -1), $sha ); - ($msg) = ( $log =~ /^\w+\s+([^\n]+)/ ); + $msg = $log =~ /^\w+\s+([^\n]+)/; } push @out, sprintf " %-35s %s %s", $branch, substr( $shas->{$branch}, 0, 8 ), $msg; From 669f6592dec72c7cb7fdc7fdbe0c1cc64592d5b6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 1 Mar 2019 17:21:29 -0500 Subject: [PATCH 16/20] Comment --- lib/ES/Repo.pm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/ES/Repo.pm b/lib/ES/Repo.pm index c302afe698b74..2dbbec6e773a3 100644 --- a/lib/ES/Repo.pm +++ b/lib/ES/Repo.pm @@ -200,6 +200,11 @@ sub extract { my ( $title, $branch, $path, $dest ) = @_; if ( exists $self->{sub_dirs}->{$branch} ) { + # Copies the $path from the subsitution diretory. It is tempting to + # just symlink the substitution directoriy into the destionation and + # call it a day and that *almost* works! The trouble is that we often + # use relative paths to include asciidoc files from other repositories + # and those relative paths don't work at all with symlinks. my $realpath = $self->{sub_dirs}->{$branch}->subdir($path); my $realdest = $dest->subdir($path)->parent; die "Can't find $realpath" unless -e $realpath; From cca7455337c44d944abd4c9e468b36206035751c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 7 Mar 2019 10:30:33 -0500 Subject: [PATCH 17/20] Merge gunk --- integtest/Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/integtest/Makefile b/integtest/Makefile index 26fbe208fa954..cf788311054f5 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -269,8 +269,6 @@ define INIT_REPO_WITH_FILE= git add . && \ git commit -m 'init' endef - # Check out the files we just built - git clone /tmp/small_all.git /tmp/small_all define GREP= # grep for a string in a file, outputting the whole file if there isn't From b27729d506140a00dacee177a0900534d3e71fd9 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 8 Mar 2019 09:02:21 -0500 Subject: [PATCH 18/20] Words --- build_docs.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_docs.pl b/build_docs.pl index 8d236921c3018..532c328f20c28 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -453,8 +453,11 @@ sub init_repos { user => $Opts->{user}, url => $Opts->{target_repo}, reference => $reference_dir, + # We can't keep the hash of the target repo because it is what stores + # the hashes in the first place! keep_hash => 0, - # intentionally not passing the tracker because we don't want to use it + # Intentionally not passing the tracker because we need to build the + # tracker from information in this repo. ); delete $child_dirs{ $target_repo->git_dir->absolute }; my $target_repo_checkout = "$temp_dir/target_repo"; From e8ed6b4c96ce6ca42ecccb5505a566d97e88b519 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 8 Mar 2019 10:24:15 -0500 Subject: [PATCH 19/20] Tmps --- build_docs.pl | 5 +- integtest/Makefile | 109 +++++++++++++++++----------------- integtest/small_conf.yaml | 7 ++- integtest/two_repos_conf.yaml | 6 +- 4 files changed, 65 insertions(+), 62 deletions(-) diff --git a/build_docs.pl b/build_docs.pl index 532c328f20c28..85dc76be5e981 100755 --- a/build_docs.pl +++ b/build_docs.pl @@ -708,9 +708,10 @@ sub pick_conf { #=================================== return 'conf.yaml' unless $Opts->{conf}; - my $conf = dir($Old_Pwd)->file($Opts->{conf}); + my $conf = file($Opts->{conf}); + $conf = dir($Old_Pwd)->file($Opts->{conf}) if $conf->is_relative; return $conf if -e $conf; - die $Opts->{conf} . " doesn't exist"; + die "$conf doesn't exist"; } #=================================== diff --git a/integtest/Makefile b/integtest/Makefile index b064be4b707ae..ba5d9ebf3a4c3 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -1,5 +1,6 @@ SHELL = /bin/bash -eux -o pipefail MAKEFLAGS += --silent +TMP = /tmp/docs_integtest/$@ # Used by the test for --all export GIT_AUTHOR_NAME=Test @@ -119,97 +120,98 @@ missing_include_fails_asciidoctor: missing_include.asciidoc .PHONY: simple_all_expected_files simple_all_expected_files: # Test the simplest possible `--all` invocation - $(call BUILD_MINIMAL_ALL,/tmp/simple_all) - $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/simple_all) + rm -rf $(TMP) + $(BUILD_MINIMAL_ALL) + $(MINIMAL_ALL_EXPECTED_FILES) .PHONY: keep_hash_expected_files keep_hash_expected_files: # Test that `--all --keep_hash` doesn't pull new updates - $(call BUILD_MINIMAL_ALL,/tmp/keep_hash) + rm -rf $(TMP) + $(BUILD_MINIMAL_ALL) # REPLACE the minimal documentation in the source repo - cp ../README.asciidoc /tmp/source/index.asciidoc - cd /tmp/source && \ + cp ../README.asciidoc $(TMP)/source/index.asciidoc + cd $(TMP)/source && \ git add . && \ git commit -m 'README' # Rebuild the docs with --keep_hash which should ignore the replacement /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo /tmp/keep_hash.git \ - --conf small_conf.yaml \ - --keep_hash | tee /tmp/out - $(call GREP,'No changes to push',/tmp/out) + --target_repo $(TMP)/dest.git \ + --conf $(TMP)/conf.yaml \ + --keep_hash | tee $(TMP)/out + $(call GREP,'No changes to push',$(TMP)/out) # We expact the same files as the minimal because we the changes that we # make shouldn't be included - $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/keep_hash) + $(MINIMAL_ALL_EXPECTED_FILES) .PHONY: sub_dir_expected_file sub_dir_expected_files: # Test that `--all --sub_dir` substitutes a directory for a branch of # a repo. + rm -rf $(TMP) # We still need to build the source repo because the script wants to fetch # it just in case we need another branch. - rm -rf /tmp/source - git init /tmp/source - cd /tmp/source && git commit --allow-empty -m "empty" + git init $(TMP)/source + cd $(TMP)/source && git commit --allow-empty -m "empty" # Setup the directory we'd like to substitute - rm -rf /tmp/to_sub - mkdir /tmp/to_sub - cp minimal.asciidoc /tmp/to_sub/index.asciidoc + mkdir $(TMP)/to_sub + cp minimal.asciidoc $(TMP)/to_sub/index.asciidoc - rm -rf /tmp/sub_dir.git - git init --bare /tmp/sub_dir.git + git init --bare $(TMP)/dest.git + sed -e 's|--tmp--|$(TMP)|' small_conf.yaml > $(TMP)/conf.yaml /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo /tmp/sub_dir.git \ - --conf small_conf.yaml \ - --sub_dir source:master:/tmp/to_sub + --target_repo $(TMP)/dest.git \ + --conf $(TMP)/conf.yaml \ + --sub_dir source:master:$(TMP)/to_sub - $(call MINIMAL_ALL_EXPECTED_FILES,/tmp/sub_dir) + $(MINIMAL_ALL_EXPECTED_FILES) +.PHONY: keep_hash_and_sub_dir_expected_files keep_hash_and_sub_dir_expected_files: # Test that `--all --keep_hash --sub_dir` keeps hashes the same for repos # not specified by --sub_dir but forces rebuilding all books that include # --sub_dir. - $(call INIT_REPO_WITH_FILE,/tmp/source1,includes_source2.asciidoc,docs/index.asciidoc) - $(call INIT_REPO_WITH_FILE,/tmp/source2,included.asciidoc,index.asciidoc) - rm -rf /tmp/keep_hash_and_sub_dir.git - git init --bare /tmp/keep_hash_and_sub_dir.git + rm -rf $(TMP) + $(call INIT_REPO_WITH_FILE,$(TMP)/source1,includes_source2.asciidoc,docs/index.asciidoc) + $(call INIT_REPO_WITH_FILE,$(TMP)/source2,included.asciidoc,index.asciidoc) + git init --bare $(TMP)/dest.git + sed 's|--tmp--|$(TMP)|' two_repos_conf.yaml > $(TMP)/conf.yaml /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo /tmp/keep_hash_and_sub_dir.git \ - --conf two_repos_conf.yaml + --target_repo $(TMP)/dest.git \ + --conf $(TMP)/conf.yaml # Move a "bad" file into source2 so we can be sure we're not picking it up - cp ../README.asciidoc /tmp/source2/index.asciidoc - cd /tmp/source2 && \ + cp ../README.asciidoc $(TMP)/source2/index.asciidoc + cd $(TMP)/source2 && \ git add . && \ git commit -m 'README' /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo /tmp/keep_hash_and_sub_dir.git \ - --conf two_repos_conf.yaml \ + --target_repo $(TMP)/dest.git \ + --conf $(TMP)/conf.yaml \ --keep_hash | tee /tmp/out $(call GREP,'No changes to push',/tmp/out) # Setup the directory we'd like to substitute - rm -rf /tmp/to_sub - mkdir -p /tmp/to_sub/docs - cp includes_source2.asciidoc /tmp/to_sub/docs/index.asciidoc - echo "extra extra extra" >> /tmp/to_sub/docs/index.asciidoc + mkdir -p $(TMP)/to_sub/docs + cp includes_source2.asciidoc $(TMP)/to_sub/docs/index.asciidoc + echo "extra extra extra" >> $(TMP)/to_sub/docs/index.asciidoc /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo /tmp/keep_hash_and_sub_dir.git \ - --conf two_repos_conf.yaml \ - --keep_hash --sub_dir source1:master:/tmp/to_sub | tee /tmp/out - $(call GREP,'Pushing changes',/tmp/out) - - git clone /tmp/keep_hash_and_sub_dir.git /tmp/keep_hash_and_sub_dir - $(call GREP,'extra extra extra',/tmp/keep_hash_and_sub_dir/html/test/current/_chapter.html) + --target_repo $(TMP)/dest.git \ + --conf $(TMP)/conf.yaml \ + --keep_hash --sub_dir source1:master:$(TMP)/to_sub | tee $(TMP)/out + $(call GREP,'Pushing changes',$(TMP)/out) + git clone $(TMP)/dest.git $(TMP)/dest + $(call GREP,'extra extra extra',$(TMP)/dest/html/test/current/_chapter.html) define GREP= # grep for a string in a file, outputting the whole file if there isn't @@ -230,34 +232,33 @@ define BUILD_MINIMAL_ALL= # Builds `--all` docs using a "minimal" source file # First build a repository to use as the source. - $(call INIT_REPO_WITH_FILE,/tmp/source,minimal.asciidoc,index.asciidoc) + $(call INIT_REPO_WITH_FILE,$(TMP)/source,minimal.asciidoc,index.asciidoc) # Initialize a bare repository that the docs build process can use as a # remote. It is used to pushing to github but it can push to a remote on # the filesystem just fine. - rm -rf $(1).git - git init --bare $(1).git + git init --bare $(TMP)/dest.git # Actually build the docs + sed 's|--tmp--|$(TMP)|' small_conf.yaml > $(TMP)/conf.yaml /docs_build/build_docs.pl --in_standard_docker --all --push \ - --target_repo $(1).git \ - --conf small_conf.yaml + --target_repo $(TMP)/dest.git \ + --conf $(TMP)/conf.yaml endef define MINIMAL_ALL_EXPECTED_FILES= # Checks that $(1).git contains the expected result of building the # "minimal" source file - git clone $(1).git $(1) - [ -s $(1)/html/branches.yaml ] - $(call GREP,'Test book',$(1)/html/index.html) - $(call GREP,'',$(1)/html/test/index.html) - [ -s $(1)/html/test/current/index.html ] + git clone $(TMP)/dest.git $(TMP)/dest + [ -s $(TMP)/dest/html/branches.yaml ] + $(call GREP,'Test book',$(TMP)/dest/html/index.html) + $(call GREP,'',$(TMP)/dest/html/test/index.html) + [ -s $(TMP)/dest/html/test/current/index.html ] endef define INIT_REPO_WITH_FILE= # Initializes the repo at $(1) and commits the file in $(2) with the # name $(3) - rm -rf $(1) git init $(1) mkdir -p $$(dirname $(1)/$(3)) cp $(2) $(1)/$(3) diff --git a/integtest/small_conf.yaml b/integtest/small_conf.yaml index 36f50aad072e5..9064474b4d751 100644 --- a/integtest/small_conf.yaml +++ b/integtest/small_conf.yaml @@ -26,13 +26,14 @@ paths: # these are big we tend to want them to be in a non-temporary but # .gitignored spot. The path is resolved relative to the root of the # docs repo. - repos: /tmp/repos/ + repos: --tmp--/repos/ # This configures all of the repositories used to build the docs repos: # Normally we use the `https://` prefix to clone from github but this file - # is for testing so we pick a repo on the filesystem. - source: /tmp/source + # is for testing so use a string that we can find with sed and replace with + # a file. + source: --tmp--/source # The title to use for the table of contents contents_title: Elastic Stack and Product Documentation diff --git a/integtest/two_repos_conf.yaml b/integtest/two_repos_conf.yaml index 9036f7dfe37b5..da5b87600bcb8 100644 --- a/integtest/two_repos_conf.yaml +++ b/integtest/two_repos_conf.yaml @@ -20,11 +20,11 @@ template: paths: build: html/ branch_tracker: html/branches.yaml - repos: /tmp/repos/ + repos: --tmp--/repos/ repos: - source1: /tmp/source1 - source2: /tmp/source2 + source1: --tmp--/source1 + source2: --tmp--/source2 contents_title: Elastic Stack and Product Documentation From 68a4220963729ba05ba769e398259db07e2ea695 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 8 Mar 2019 13:50:54 -0500 Subject: [PATCH 20/20] More make --- integtest/Makefile | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/integtest/Makefile b/integtest/Makefile index ba5d9ebf3a4c3..4729807f458c9 100644 --- a/integtest/Makefile +++ b/integtest/Makefile @@ -18,10 +18,11 @@ check: \ experimental_expected_files experimental_same_files \ missing_include_fails_asciidoc missing_include_fails_asciidoctor \ readme_expected_files readme_same_files \ - simple_all_expected_files \ - keep_hash_expected_files \ - sub_dir_expected_files \ - keep_hash_and_sub_dir_expected_files + simple_all \ + relative_conf_file \ + keep_hash \ + sub_dir \ + keep_hash_and_sub_dir .PHONY: style style: html_diff @@ -117,15 +118,26 @@ missing_include_fails_asciidoctor: missing_include.asciidoc /tmp/%_asciidoctor: $(BD) --asciidoctor --doc $*.asciidoc -.PHONY: simple_all_expected_files -simple_all_expected_files: +.PHONY: simple_all +simple_all: # Test the simplest possible `--all` invocation rm -rf $(TMP) $(BUILD_MINIMAL_ALL) $(MINIMAL_ALL_EXPECTED_FILES) -.PHONY: keep_hash_expected_files -keep_hash_expected_files: +.PHONY: relative_conf_file +relative_conf_file: + # Make sure that using a relative referece to the --conf file works. + rm -rf $(TMP) + $(SETUP_MINIMAL_ALL) + cd $(TMP) && \ + /docs_build/build_docs.pl --in_standard_docker --all --push \ + --target_repo $(TMP)/dest.git \ + --conf conf.yaml + $(MINIMAL_ALL_EXPECTED_FILES) + +.PHONY: keep_hash +keep_hash: # Test that `--all --keep_hash` doesn't pull new updates rm -rf $(TMP) $(BUILD_MINIMAL_ALL) @@ -147,8 +159,8 @@ keep_hash_expected_files: # make shouldn't be included $(MINIMAL_ALL_EXPECTED_FILES) -.PHONY: sub_dir_expected_file -sub_dir_expected_files: +.PHONY: sub_dir +sub_dir: # Test that `--all --sub_dir` substitutes a directory for a branch of # a repo. rm -rf $(TMP) @@ -171,8 +183,8 @@ sub_dir_expected_files: $(MINIMAL_ALL_EXPECTED_FILES) -.PHONY: keep_hash_and_sub_dir_expected_files -keep_hash_and_sub_dir_expected_files: +.PHONY: keep_hash_and_sub_dir +keep_hash_and_sub_dir: # Test that `--all --keep_hash --sub_dir` keeps hashes the same for repos # not specified by --sub_dir but forces rebuilding all books that include # --sub_dir. @@ -228,9 +240,7 @@ define GREP= } endef -define BUILD_MINIMAL_ALL= - # Builds `--all` docs using a "minimal" source file - +define SETUP_MINIMAL_ALL= # First build a repository to use as the source. $(call INIT_REPO_WITH_FILE,$(TMP)/source,minimal.asciidoc,index.asciidoc) @@ -241,6 +251,11 @@ define BUILD_MINIMAL_ALL= # Actually build the docs sed 's|--tmp--|$(TMP)|' small_conf.yaml > $(TMP)/conf.yaml +endef + +define BUILD_MINIMAL_ALL= + # Builds `--all` docs using a "minimal" source file + $(SETUP_MINIMAL_ALL) /docs_build/build_docs.pl --in_standard_docker --all --push \ --target_repo $(TMP)/dest.git \ --conf $(TMP)/conf.yaml