From 16385e65249d751942ac75be9871ffe6e1d90379 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 27 Oct 2016 22:57:28 +0200 Subject: [PATCH 1/6] Add argument to enhance portability: --skip-cache-invalidation --- build_docs.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/build_docs.py b/build_docs.py index 43bab5f..79bce65 100644 --- a/build_docs.py +++ b/build_docs.py @@ -63,7 +63,8 @@ def shell_out(cmd): raise -def build_one(version, isdev, quick, sphinxbuild, build_root, www_root): +def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, + skip_cache_invalidation=False): checkout = build_root + "/python" + str(version).replace('.', '') target = www_root + "/" + str(version) logging.info("Doc autobuild started in %s", checkout) @@ -107,7 +108,7 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root): changed.append("archives/" + fn) logging.info("%s files changed", len(changed)) - if changed: + if changed and not skip_cache_invalidation: target_ino = os.stat(target).st_ino targets_dir = os.path.dirname(target) prefixes = [] @@ -123,12 +124,15 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root): logging.info("Finished %s", checkout) -def build_devguide(devguide_checkout, devguide_target, sphinxbuild): +def build_devguide(devguide_checkout, devguide_target, sphinxbuild, + skip_cache_invalidation=False): logging.info("Building devguide") shell_out("git -C %s pull" % (devguide_checkout,)) shell_out("%s %s %s" % (sphinxbuild, devguide_checkout, devguide_target)) shell_out("chmod -R o+r %s" % (devguide_target,)) - # TODO Do Fastly invalidation. + if not skip_cache_invalidation: + # TODO Do Fastly invalidation. + pass def parse_args(): @@ -164,6 +168,10 @@ def parse_args(): "--devguide-target", help="Path where the generated devguide should be copied.", default="/srv/docs.python.org/devguide") + parser.add_argument( + "--skip-cache-invalidation", + help="Skip fastly cache invalidation.", + action="store_true") return parser.parse_args() @@ -180,11 +188,14 @@ def parse_args(): try: if args.branch: build_one(args.branch, args.devel, args.quick, sphinxbuild, - args.build_root, args.www_root) + args.build_root, args.www_root, + args.skip_cache_invalidation) else: for version, devel in BRANCHES: build_one(version, devel, args.quick, sphinxbuild, - args.build_root, args.www_root) - build_devguide(args.devguide_checkout, args.devguide_target, sphinxbuild) + args.build_root, args.www_root, + args.skip_cache_invalidation) + build_devguide(args.devguide_checkout, args.devguide_target, + sphinxbuild, args.skip_cache_invalidation) except Exception: logging.exception("docs build raised exception") From 4498f5f4ebb6daeb7d05860dcbc23dec59fecb34 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 27 Oct 2016 23:01:09 +0200 Subject: [PATCH 2/6] Add argument to enhance portability: --group --- build_docs.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/build_docs.py b/build_docs.py index 79bce65..61d83cb 100644 --- a/build_docs.py +++ b/build_docs.py @@ -64,7 +64,7 @@ def shell_out(cmd): def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, - skip_cache_invalidation=False): + skip_cache_invalidation=False, group='docs'): checkout = build_root + "/python" + str(version).replace('.', '') target = www_root + "/" + str(version) logging.info("Doc autobuild started in %s", checkout) @@ -92,16 +92,16 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, changed.append(rel_path) logging.info("Copying HTML files to %s", target) - shell_out("chown -R :docs Doc/build/html/") + shell_out("chown -R :{} Doc/build/html/".format(group)) shell_out("chmod -R o+r Doc/build/html/") shell_out("find Doc/build/html/ -type d -exec chmod o+x {} ';'") shell_out("cp -a Doc/build/html/* %s" % target) if not quick: logging.debug("Copying dist files") - shell_out("chown -R :docs Doc/dist/") + shell_out("chown -R :{} Doc/dist/".format(group)) shell_out("chmod -R o+r Doc/dist/") shell_out("mkdir -m o+rx -p %s/archives" % target) - shell_out("chown :docs %s/archives" % target) + shell_out("chown :{} {}/archives".format(group, target)) shell_out("cp -a Doc/dist/* %s/archives" % target) changed.append("archives/") for fn in os.listdir(os.path.join(target, "archives")): @@ -172,6 +172,10 @@ def parse_args(): "--skip-cache-invalidation", help="Skip fastly cache invalidation.", action="store_true") + parser.add_argument( + "--group", + help="Group files on targets and www-root file should get.", + default="docs") return parser.parse_args() @@ -189,12 +193,13 @@ def parse_args(): if args.branch: build_one(args.branch, args.devel, args.quick, sphinxbuild, args.build_root, args.www_root, - args.skip_cache_invalidation) + args.skip_cache_invalidation, + args.group) else: for version, devel in BRANCHES: build_one(version, devel, args.quick, sphinxbuild, args.build_root, args.www_root, - args.skip_cache_invalidation) + args.skip_cache_invalidation, args.group) build_devguide(args.devguide_checkout, args.devguide_target, sphinxbuild, args.skip_cache_invalidation) except Exception: From 1c25f00cbe60e918d4235694be0e16e9928d7901 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Thu, 27 Oct 2016 23:18:46 +0200 Subject: [PATCH 3/6] Add argument to enhance portability: --git --- build_docs.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 61d83cb..3fea43a 100644 --- a/build_docs.py +++ b/build_docs.py @@ -64,14 +64,18 @@ def shell_out(cmd): def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, - skip_cache_invalidation=False, group='docs'): + skip_cache_invalidation=False, group='docs', git=False): checkout = build_root + "/python" + str(version).replace('.', '') target = www_root + "/" + str(version) logging.info("Doc autobuild started in %s", checkout) os.chdir(checkout) logging.info("Updating checkout") - shell_out("hg pull -u") + if git: + shell_out("git reset --hard HEAD") + shell_out("git pull --ff-only") + else: + shell_out("hg pull -u") maketarget = "autobuild-" + ("dev" if isdev else "stable") + ("-html" if quick else "") logging.info("Running make %s", maketarget) @@ -176,6 +180,10 @@ def parse_args(): "--group", help="Group files on targets and www-root file should get.", default="docs") + parser.add_argument( + "--git", + help="Use git instead of mercurial.", + action="store_true") return parser.parse_args() @@ -194,12 +202,12 @@ def parse_args(): build_one(args.branch, args.devel, args.quick, sphinxbuild, args.build_root, args.www_root, args.skip_cache_invalidation, - args.group) + args.group, args.git) else: for version, devel in BRANCHES: build_one(version, devel, args.quick, sphinxbuild, args.build_root, args.www_root, - args.skip_cache_invalidation, args.group) + args.skip_cache_invalidation, args.group, args.git) build_devguide(args.devguide_checkout, args.devguide_target, sphinxbuild, args.skip_cache_invalidation) except Exception: From b49be2a3864fd1f61df6cef90045fc8a98c4d080 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 28 Oct 2016 00:06:40 +0200 Subject: [PATCH 4/6] Add argument to enhance portability: --log-directory --- build_docs.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/build_docs.py b/build_docs.py index 3fea43a..e9050a3 100644 --- a/build_docs.py +++ b/build_docs.py @@ -64,7 +64,8 @@ def shell_out(cmd): def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, - skip_cache_invalidation=False, group='docs', git=False): + skip_cache_invalidation=False, group='docs', git=False, + log_directory='/var/log/docsbuild/'): checkout = build_root + "/python" + str(version).replace('.', '') target = www_root + "/" + str(version) logging.info("Doc autobuild started in %s", checkout) @@ -80,8 +81,8 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, maketarget = "autobuild-" + ("dev" if isdev else "stable") + ("-html" if quick else "") logging.info("Running make %s", maketarget) logname = os.path.basename(checkout) + ".log" - shell_out("cd Doc; make SPHINXBUILD=%s %s >> /var/log/docsbuild/%s 2>&1" % - (sphinxbuild, maketarget, logname)) + shell_out("cd Doc; make SPHINXBUILD=%s %s >> %s 2>&1" % + (sphinxbuild, maketarget, os.path.join(log_directory, logname))) logging.info("Computing changed files") changed = [] @@ -184,30 +185,36 @@ def parse_args(): "--git", help="Use git instead of mercurial.", action="store_true") + parser.add_argument( + "--log-directory", + help="Directory used to store logs.", + default="/var/log/docsbuild/") return parser.parse_args() if __name__ == '__main__': + args = parse_args() if sys.stderr.isatty(): logging.basicConfig(format="%(levelname)s:%(message)s", stream=sys.stderr) else: logging.basicConfig(format="%(levelname)s:%(asctime)s:%(message)s", - filename="/var/log/docsbuild/docsbuild.log") + filename=os.path.join(args.log_directory, + "docsbuild.log")) logging.root.setLevel(logging.DEBUG) - args = parse_args() sphinxbuild = os.path.join(args.build_root, "environment/bin/sphinx-build") try: if args.branch: build_one(args.branch, args.devel, args.quick, sphinxbuild, args.build_root, args.www_root, args.skip_cache_invalidation, - args.group, args.git) + args.group, args.git, args.log_directory) else: for version, devel in BRANCHES: build_one(version, devel, args.quick, sphinxbuild, args.build_root, args.www_root, - args.skip_cache_invalidation, args.group, args.git) + args.skip_cache_invalidation, args.group, args.git, + args.log_directory) build_devguide(args.devguide_checkout, args.devguide_target, sphinxbuild, args.skip_cache_invalidation) except Exception: From 1a6bcd3f64b0b13fd1578f3b0bf5096df11ac5bf Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 28 Oct 2016 00:27:00 +0200 Subject: [PATCH 5/6] Extract changed files to reuse it in devguide. --- build_docs.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/build_docs.py b/build_docs.py index e9050a3..4eb9053 100644 --- a/build_docs.py +++ b/build_docs.py @@ -63,6 +63,23 @@ def shell_out(cmd): raise +def changed_files(directory, other): + logging.info("Computing changed files") + changed = [] + if directory[-1] != '/': + directory += '/' + for dirpath, dirnames, filenames in os.walk(directory): + dir_rel = dirpath[len(directory):] + for fn in filenames: + local_path = os.path.join(dirpath, fn) + rel_path = os.path.join(dir_rel, fn) + target_path = os.path.join(other, rel_path) + if (os.path.exists(target_path) and + not _file_unchanged(target_path, local_path)): + changed.append(rel_path) + return changed + + def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, skip_cache_invalidation=False, group='docs', git=False, log_directory='/var/log/docsbuild/'): @@ -84,18 +101,7 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, shell_out("cd Doc; make SPHINXBUILD=%s %s >> %s 2>&1" % (sphinxbuild, maketarget, os.path.join(log_directory, logname))) - logging.info("Computing changed files") - changed = [] - for dirpath, dirnames, filenames in os.walk("Doc/build/html/"): - dir_rel = dirpath[len("Doc/build/html/"):] - for fn in filenames: - local_path = os.path.join(dirpath, fn) - rel_path = os.path.join(dir_rel, fn) - target_path = os.path.join(target, rel_path) - if (os.path.exists(target_path) and - not _file_unchanged(target_path, local_path)): - changed.append(rel_path) - + changed = changed_files(os.path.join(checkout, "Doc/build/html"), target) logging.info("Copying HTML files to %s", target) shell_out("chown -R :{} Doc/build/html/".format(group)) shell_out("chmod -R o+r Doc/build/html/") From 98d445490ea3b092e51d857d5fc153de1fdf3f43 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Fri, 28 Oct 2016 01:07:12 +0200 Subject: [PATCH 6/6] Implemente fastly invalidation for devguide. --- build_docs.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/build_docs.py b/build_docs.py index 4eb9053..5e5e5c4 100644 --- a/build_docs.py +++ b/build_docs.py @@ -137,13 +137,20 @@ def build_one(version, isdev, quick, sphinxbuild, build_root, www_root, def build_devguide(devguide_checkout, devguide_target, sphinxbuild, skip_cache_invalidation=False): + build_directory = os.path.join(devguide_checkout, "build/html") logging.info("Building devguide") shell_out("git -C %s pull" % (devguide_checkout,)) - shell_out("%s %s %s" % (sphinxbuild, devguide_checkout, devguide_target)) + shell_out("%s %s %s" % (sphinxbuild, devguide_checkout, build_directory)) + changed = changed_files(build_directory, devguide_target) + shell_out("mkdir -p {}".format(devguide_target)) + shell_out("cp -a {}/* {}".format(build_directory, devguide_target)) shell_out("chmod -R o+r %s" % (devguide_target,)) - if not skip_cache_invalidation: - # TODO Do Fastly invalidation. - pass + if changed and not skip_cache_invalidation: + prefix = os.path.basename(devguide_target) + to_purge = [prefix] + to_purge.extend(prefix + "/" + p for p in changed) + logging.info("Running CDN purge") + shell_out("curl -X PURGE \"https://docs.python.org/{%s}\"" % ",".join(to_purge)) def parse_args():