From 8a9ac6cbd507a6850a5a0012a4627c09cf9cd7d6 Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Mon, 26 Oct 2015 23:20:50 +1100 Subject: [PATCH] Converted to TypeScript --- .gitignore | 1 + examples/readme-caps.js | 101 ++-- examples/readme-caps.ts | 78 +++ lib/xhr-node.js | 158 +++--- lib/xhr-node.ts | 81 ++++ lib/xhr.js | 133 +++--- lib/xhr.ts | 72 +++ mixins/github-db.js | 1002 +++++++++++++++++++-------------------- mixins/github-db.ts | 576 ++++++++++++++++++++++ tsconfig.json | 31 ++ tsd.json | 12 + 11 files changed, 1519 insertions(+), 726 deletions(-) create mode 100644 .gitignore create mode 100644 examples/readme-caps.ts create mode 100644 lib/xhr-node.ts create mode 100644 lib/xhr.ts create mode 100644 mixins/github-db.ts create mode 100644 tsconfig.json create mode 100644 tsd.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a2fafbf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +typings/ diff --git a/examples/readme-caps.js b/examples/readme-caps.js index 8eb8c62..66df250 100644 --- a/examples/readme-caps.js +++ b/examples/readme-caps.js @@ -1,78 +1,37 @@ -var repo = {}; - -// This only works for normal repos. Github doesn't allow access to gists as -// far as I can tell. -var githubName = "creationix/js-github"; - -// Your user can generate these manually at https://github.com/settings/tokens/new -// Or you can use an oauth flow to get a token for the user. -var githubToken = "8fe7e5ad65814ea315daad99b6b65f2fd0e4c5aa"; - -// Mixin the main library using github to provide the following: -// - repo.loadAs(type, hash) => value -// - repo.saveAs(type, value) => hash -// - repo.readRef(ref) => hash -// - repo.updateRef(ref, hash) => hash -// - repo.createTree(entries) => hash -// - repo.hasHash(hash) => has +let repo = {}; +const githubName = "creationix/js-github"; +const githubToken = "8fe7e5ad65814ea315daad99b6b65f2fd0e4c5aa"; require('../mixins/github-db')(repo, githubName, githubToken); - - -// Github has this built-in, but it's currently very buggy so we replace with -// the manual implementation in js-git. require('js-git/mixins/create-tree')(repo); - -// Cache everything except blobs over 100 bytes in memory. -// This makes path-to-hash lookup a sync operation in most cases. require('js-git/mixins/mem-cache')(repo); - -// Combine concurrent read requests for the same hash require('js-git/mixins/read-combiner')(repo); - -// Add in value formatting niceties. Also adds text and array types. require('js-git/mixins/formats')(repo); - -// I'm using generator syntax, but callback style also works. -// See js-git main docs for more details. -var run = require('gen-run'); +let run = require('gen-run'); run(function* () { - var headHash = yield repo.readRef("refs/heads/master"); - var commit = yield repo.loadAs("commit", headHash); - var tree = yield repo.loadAs("tree", commit.tree); - var entry = tree["README.md"]; - var readme = yield repo.loadAs("text", entry.hash); - - // Build the updates array - var updates = [ - { - path: "README.md", // Update the existing entry - mode: entry.mode, // Preserve the mode (it might have been executible) - content: readme.toUpperCase() // Write the new content - } - ]; - // Based on the existing tree, we only want to update, not replace. - updates.base = commit.tree; - - // Create the new file and the updated tree. - var treeHash = yield repo.createTree(updates); - - var commitHash = yield repo.saveAs("commit", { - tree: treeHash, - author: { - name: "Tim Caswell", - email: "tim@creationix.com" - }, - parent: headHash, - message: "Change README.md to be all uppercase using js-github" - }); - - // Now we can browse to this commit by hash, but it's still not in master. - // We need to update the ref to point to this new commit. - console.log("COMMIT", commitHash) - - // Save it to a new branch (Or update existing one) - var new_hash = yield repo.updateRef("refs/heads/new-branch", commitHash); - - // And delete this new branch: - yield repo.deleteRef("refs/heads/new-branch"); + const headHash = yield repo.readRef("refs/heads/master"); + const commit = yield repo.loadAs("commit", headHash); + const tree = yield repo.loadAs("tree", commit.tree); + const entry = tree["README.md"]; + const readme = yield repo.loadAs("text", entry.hash); + let updates = [ + { + path: "README.md", + mode: entry.mode, + content: readme.toUpperCase() + } + ]; + updates.base = commit.tree; + const treeHash = yield repo.createTree(updates); + const commitHash = yield repo.saveAs("commit", { + tree: treeHash, + author: { + name: "Tim Caswell", + email: "tim@creationix.com" + }, + parent: headHash, + message: "Change README.md to be all uppercase using js-github" + }); + console.log("COMMIT", commitHash); + const new_hash = yield repo.updateRef("refs/heads/new-branch", commitHash); + yield repo.deleteRef("refs/heads/new-branch"); }); diff --git a/examples/readme-caps.ts b/examples/readme-caps.ts new file mode 100644 index 0000000..2eada1e --- /dev/null +++ b/examples/readme-caps.ts @@ -0,0 +1,78 @@ +let repo: any = {}; + +// This only works for normal repos. Github doesn't allow access to gists as +// far as I can tell. +const githubName = "creationix/js-github"; + +// Your user can generate these manually at https://github.com/settings/tokens/new +// Or you can use an oauth flow to get a token for the user. +const githubToken = "8fe7e5ad65814ea315daad99b6b65f2fd0e4c5aa"; + +// Mixin the main library using github to provide the following: +// - repo.loadAs(type, hash) => value +// - repo.saveAs(type, value) => hash +// - repo.readRef(ref) => hash +// - repo.updateRef(ref, hash) => hash +// - repo.createTree(entries) => hash +// - repo.hasHash(hash) => has +require('../mixins/github-db')(repo, githubName, githubToken); + + +// Github has this built-in, but it's currently very buggy so we replace with +// the manual implementation in js-git. +require('js-git/mixins/create-tree')(repo); + +// Cache everything except blobs over 100 bytes in memory. +// This makes path-to-hash lookup a sync operation in most cases. +require('js-git/mixins/mem-cache')(repo); + +// Combine concurrent read requests for the same hash +require('js-git/mixins/read-combiner')(repo); + +// Add in value formatting niceties. Also adds text and array types. +require('js-git/mixins/formats')(repo); + +// I'm using generator syntax, but callback style also works. +// See js-git main docs for more details. +let run = require('gen-run'); +run(function* () { + const headHash = yield repo.readRef("refs/heads/master"); + const commit = yield repo.loadAs("commit", headHash); + const tree = yield repo.loadAs("tree", commit.tree); + const entry = tree["README.md"]; + const readme = yield repo.loadAs("text", entry.hash); + + // Build the updates array + let updates: any = [ + { + path: "README.md", // Update the existing entry + mode: entry.mode, // Preserve the mode (it might have been executible) + content: readme.toUpperCase() // Write the new content + } + ]; + // Based on the existing tree, we only want to update, not replace. + updates.base = commit.tree; + + // Create the new file and the updated tree. + const treeHash = yield repo.createTree(updates); + + const commitHash = yield repo.saveAs("commit", { + tree: treeHash, + author: { + name: "Tim Caswell", + email: "tim@creationix.com" + }, + parent: headHash, + message: "Change README.md to be all uppercase using js-github" + }); + + // Now we can browse to this commit by hash, but it's still not in master. + // We need to update the ref to point to this new commit. + console.log("COMMIT", commitHash) + + // Save it to a new branch (Or update existing one) + const new_hash = yield repo.updateRef("refs/heads/new-branch", commitHash); + + // And delete this new branch: + yield repo.deleteRef("refs/heads/new-branch"); +}); diff --git a/lib/xhr-node.js b/lib/xhr-node.js index 57edd5c..43cd94f 100644 --- a/lib/xhr-node.js +++ b/lib/xhr-node.js @@ -1,81 +1,83 @@ -var https = require('https'); -var statusCodes = require('http').STATUS_CODES; -var urlParse = require('url').parse; -var path = require('path'); - -module.exports = function (root, accessToken, githubHostname) { - var cache = {}; - githubHostname = (githubHostname || "https://api.github.com/"); - return function request(method, url, body, callback) { - if (typeof body === "function" && callback === undefined) { - callback = body; - body = undefined; - } - if (!callback) return request.bind(this, accessToken, method, url, body); - url = url.replace(":root", root); - - var json; - var headers = { - "User-Agent": "node.js" - }; - if (accessToken) { - headers["Authorization"] = "token " + accessToken; - } - if (body) { - headers["Content-Type"] = "application/json"; - try { json = new Buffer(JSON.stringify(body)); } - catch (err) { return callback(err); } - headers["Content-Length"] = json.length; - } - if (method === "GET") { - var cached = cache[url]; - if (cached) { - headers["If-None-Match"] = cached.etag; - } - } - - var urlparts = urlParse(githubHostname); - var options = { - hostname: urlparts.hostname, - path: path.join(urlparts.pathname, url), - method: method, - headers: headers - }; - - var req = https.request(options, function (res) { - var body = []; - res.on("data", function (chunk) { - body.push(chunk); - }); - res.on("end", function () { - body = Buffer.concat(body).toString(); - console.log(method, url, res.statusCode); - console.log("Rate limit %s/%s left", res.headers['x-ratelimit-remaining'], res.headers['x-ratelimit-limit']); - //console.log(body); - if (res.statusCode === 200 && method === "GET" && /\/refs\//.test(url)) { - cache[url] = { - body: body, - etag: res.headers.etag - }; +let https = require('https'); +let statusCodes = require('http').STATUS_CODES; +let urlParse = require('url').parse; +let path = require('path'); +export function xhrNode(root, accessToken, githubHostname) { + let cache = {}; + githubHostname = githubHostname || "https://api.github.com/"; + return function request(method, url, body, callback) { + if (typeof body === "function" && callback === undefined) { + callback = body; + body = undefined; } - else if (res.statusCode === 304) { - body = cache[url].body; - res.statusCode = 200; - } - // Fake parts of the xhr object using node APIs - var xhr = { - status: res.statusCode, - statusText: res.statusCode + " " + statusCodes[res.statusCode] + if (!callback) + return request.bind(this, accessToken, method, url, body); + url = url.replace(":root", root); + let json; + let headers = { + "User-Agent": "node.js" }; - var response = {message:body}; - if (body){ - try { response = JSON.parse(body); } - catch (err) {} + if (accessToken) { + headers["Authorization"] = "token " + accessToken; + } + if (body) { + headers["Content-Type"] = "application/json"; + try { + json = new Buffer(JSON.stringify(body)); + } + catch (err) { + return callback(err); + } + headers["Content-Length"] = json.length; } - return callback(null, xhr, response); - }); - }); - req.end(json); - req.on("error", callback); - }; -}; + if (method === "GET") { + let cached = cache[url]; + if (cached) { + headers["If-None-Match"] = cached.etag; + } + } + let urlparts = urlParse(githubHostname); + let options = { + hostname: urlparts.hostname, + path: path.join(urlparts.pathname, url), + method: method, + headers: headers + }; + let req = https.request(options, function (res) { + let body = []; + res.on("data", function (chunk) { + body.push(chunk); + }); + res.on("end", function () { + body = Buffer.concat(body).toString(); + console.log(method, url, res.statusCode); + console.log("Rate limit %s/%s left", res.headers['x-ratelimit-remaining'], res.headers['x-ratelimit-limit']); + if (res.statusCode === 200 && method === "GET" && /\/refs\//.test(url)) { + cache[url] = { + body: body, + etag: res.headers.etag + }; + } + else if (res.statusCode === 304) { + body = cache[url].body; + res.statusCode = 200; + } + let xhr = { + status: res.statusCode, + statusText: res.statusCode + " " + statusCodes[res.statusCode] + }; + let response = { message: body }; + if (body) { + try { + response = JSON.parse(body); + } + catch (err) { } + } + return callback(null, xhr, response); + }); + }); + req.end(json); + req.on("error", callback); + }; +} +; diff --git a/lib/xhr-node.ts b/lib/xhr-node.ts new file mode 100644 index 0000000..4ebb881 --- /dev/null +++ b/lib/xhr-node.ts @@ -0,0 +1,81 @@ +let https = require('https'); +let statusCodes = require('http').STATUS_CODES; +let urlParse = require('url').parse; +let path = require('path'); + +export function xhrNode(root, accessToken, githubHostname) { + let cache = {}; + githubHostname = githubHostname || "https://api.github.com/"; + return function request(method, url, body, callback) { + if (typeof body === "function" && callback === undefined) { + callback = body; + body = undefined; + } + if (!callback) return request.bind(this, accessToken, method, url, body); + url = url.replace(":root", root); + + let json: Buffer; + let headers = { + "User-Agent": "node.js" + }; + if (accessToken) { + headers["Authorization"] = "token " + accessToken; + } + if (body) { + headers["Content-Type"] = "application/json"; + try { json = new Buffer(JSON.stringify(body)); } + catch (err) { return callback(err); } + headers["Content-Length"] = json.length; + } + if (method === "GET") { + let cached = cache[url]; + if (cached) { + headers["If-None-Match"] = cached.etag; + } + } + + let urlparts = urlParse(githubHostname); + let options = { + hostname: urlparts.hostname, + path: path.join(urlparts.pathname, url), + method: method, + headers: headers + }; + + let req = https.request(options, function(res) { + let body: any = []; + res.on("data", function(chunk) { + body.push(chunk); + }); + res.on("end", function() { + body = Buffer.concat(body).toString(); + console.log(method, url, res.statusCode); + console.log("Rate limit %s/%s left", res.headers['x-ratelimit-remaining'], res.headers['x-ratelimit-limit']); + //console.log(body); + if (res.statusCode === 200 && method === "GET" && /\/refs\//.test(url)) { + cache[url] = { + body: body, + etag: res.headers.etag + }; + } + else if (res.statusCode === 304) { + body = cache[url].body; + res.statusCode = 200; + } + // Fake parts of the xhr object using node APIs + let xhr = { + status: res.statusCode, + statusText: res.statusCode + " " + statusCodes[res.statusCode] + }; + let response = { message: body }; + if (body) { + try { response = JSON.parse(body); } + catch (err) { } + } + return callback(null, xhr, response); + }); + }); + req.end(json); + req.on("error", callback); + }; +}; diff --git a/lib/xhr.js b/lib/xhr.js index 593b809..28a56e7 100644 --- a/lib/xhr.js +++ b/lib/xhr.js @@ -1,71 +1,72 @@ -"use strict"; - -var isNode = typeof process === 'object' && - typeof process.versions === 'object' && - process.versions.node && - process.__atom_type !== "renderer"; - -// Node.js https module +const isNode = typeof process === 'object' && + typeof process.versions === 'object' && + process.versions.node && + process['__atom_type'] !== "renderer"; if (isNode) { - var nodeRequire = require; // Prevent mine.js from seeing this require - module.exports = nodeRequire('./xhr-node.js'); + const nodeRequire = require; + exports = nodeRequire('./xhr-node.js'); } - -// Browser XHR else { - module.exports = function (root, accessToken, githubHostname) { - var timeout = 2000; - githubHostname = (githubHostname || 'https://api.github.com'); - return request; - - function request(method, url, body, callback) { - if (typeof body === "function") { - callback = body; - body = undefined; - } - else if (!callback) return request.bind(null, method, url, body); - url = url.replace(":root", root); - var done = false; - var json; - var xhr = new XMLHttpRequest(); - xhr.timeout = timeout; - xhr.open(method, githubHostname + url, true); - xhr.setRequestHeader("Authorization", "token " + accessToken); - if (body) { - try { json = JSON.stringify(body); } - catch (err) { return callback(err); } - } - xhr.ontimeout = onTimeout; - xhr.onerror = function() { - callback(new Error("Error requesting " + url)); - }; - xhr.onreadystatechange = onReadyStateChange; - xhr.send(json); - - function onReadyStateChange() { - if (done) return; - if (xhr.readyState !== 4) return; - // Give onTimeout a chance to run first if that's the reason status is 0. - if (!xhr.status) return setTimeout(onReadyStateChange, 0); - done = true; - var response = {message:xhr.responseText}; - if (xhr.responseText){ - try { response = JSON.parse(xhr.responseText); } - catch (err) {} + exports = function (root, accessToken, githubHostname) { + var timeout = 2000; + githubHostname = (githubHostname || 'https://api.github.com'); + return request; + function request(method, url, body, callback) { + if (typeof body === "function") { + callback = body; + body = undefined; + } + else if (!callback) + return request.bind(null, method, url, body); + url = url.replace(":root", root); + var done = false; + var json; + var xhr = new XMLHttpRequest(); + xhr.timeout = timeout; + xhr.open(method, githubHostname + url, true); + xhr.setRequestHeader("Authorization", "token " + accessToken); + if (body) { + try { + json = JSON.stringify(body); + } + catch (err) { + return callback(err); + } + } + xhr.ontimeout = onTimeout; + xhr.onerror = function () { + callback(new Error("Error requesting " + url)); + }; + xhr.onreadystatechange = onReadyStateChange; + xhr.send(json); + function onReadyStateChange() { + if (done) + return; + if (xhr.readyState !== 4) + return; + if (!xhr.status) + return setTimeout(onReadyStateChange, 0); + done = true; + var response = { message: xhr.responseText }; + if (xhr.responseText) { + try { + response = JSON.parse(xhr.responseText); + } + catch (err) { } + } + xhr.responseBody = response; + return callback(null, xhr, response); + } + function onTimeout() { + if (done) + return; + if (timeout < 8000) { + timeout *= 2; + return request(method, url, body, callback); + } + done = true; + callback(new Error("Timeout requesting " + url)); + } } - xhr.body = response; - return callback(null, xhr, response); - } - - function onTimeout() { - if (done) return; - if (timeout < 8000) { - timeout *= 2; - return request(method, url, body, callback); - } - done = true; - callback(new Error("Timeout requesting " + url)); - } - } - }; + }; } diff --git a/lib/xhr.ts b/lib/xhr.ts new file mode 100644 index 0000000..c5df615 --- /dev/null +++ b/lib/xhr.ts @@ -0,0 +1,72 @@ +/// + +const isNode: boolean = typeof process === 'object' && + typeof process.versions === 'object' && + process.versions.node && + process['__atom_type'] !== "renderer"; + +// Node.js https module +if (isNode) { + const nodeRequire = require; // Prevent mine.js from seeing this require + exports = nodeRequire('./xhr-node.js'); +} + +// Browser XHR +else { + exports = function(root, accessToken, githubHostname): (method: string, url: string, body: string | any, callback?) => any { + var timeout = 2000; + githubHostname = (githubHostname || 'https://api.github.com'); + return request; + + function request(method: string, url: string, body: string | any, callback?) { + if (typeof body === "function") { + callback = body; + body = undefined; + } + else if (!callback) return request.bind(null, method, url, body); + url = url.replace(":root", root); + var done = false; + var json; + var xhr: XMLHttpRequest = new XMLHttpRequest(); + xhr.timeout = timeout; + xhr.open(method, githubHostname + url, true); + xhr.setRequestHeader("Authorization", "token " + accessToken); + if (body) { + try { json = JSON.stringify(body); } + catch (err) { return callback(err); } + } + xhr.ontimeout = onTimeout; + xhr.onerror = function() { + callback(new Error("Error requesting " + url)); + }; + xhr.onreadystatechange = onReadyStateChange; + xhr.send(json); + + function onReadyStateChange() { + if (done) return; + if (xhr.readyState !== 4) return; + // Give onTimeout a chance to run first if that's the reason status is 0. + if (!xhr.status) return setTimeout(onReadyStateChange, 0); + done = true; + var response = { message: xhr.responseText }; + if (xhr.responseText) { + try { response = JSON.parse(xhr.responseText); } + catch (err) { } + } + xhr.responseBody = response; + //xhr.body = response; + return callback(null, xhr, response); + } + + function onTimeout() { + if (done) return; + if (timeout < 8000) { + timeout *= 2; + return request(method, url, body, callback); + } + done = true; + callback(new Error("Timeout requesting " + url)); + } + } + }; +} diff --git a/mixins/github-db.js b/mixins/github-db.js index 7440308..a41e56e 100644 --- a/mixins/github-db.js +++ b/mixins/github-db.js @@ -1,576 +1,556 @@ "use strict"; - var modes = require('js-git/lib/modes'); var xhr = require('../lib/xhr'); var bodec = require('bodec'); var sha1 = require('git-sha1'); var frame = require('js-git/lib/object-codec').frame; - -var modeToType = { - "040000": "tree", - "100644": "blob", // normal file - "100755": "blob", // executable file - "120000": "blob", // symlink - "160000": "commit" // gitlink +const modeToType = { + "040000": "tree", + "100644": "blob", + "100755": "blob", + "120000": "blob", + "160000": "commit" }; - -var encoders = { - commit: encodeCommit, - tag: encodeTag, - tree: encodeTree, - blob: encodeBlob +const encoders = { + commit: encodeCommit, + tag: encodeTag, + tree: encodeTree, + blob: encodeBlob }; - -var decoders = { - commit: decodeCommit, - tag: decodeTag, - tree: decodeTree, - blob: decodeBlob, +const decoders = { + commit: decodeCommit, + tag: decodeTag, + tree: decodeTree, + blob: decodeBlob, }; - -var typeCache = {}; - -// Precompute hashes for empty blob and empty tree since github won't -var empty = bodec.create(0); -var emptyBlob = sha1(frame({ type: "blob", body: empty })); -var emptyTree = sha1(frame({ type: "tree", body: empty })); - -// Implement the js-git object interface using github APIs +let typeCache = {}; +const empty = bodec.create(0); +const emptyBlob = sha1(frame({ type: "blob", body: empty })); +const emptyTree = sha1(frame({ type: "tree", body: empty })); module.exports = function (repo, root, accessToken, githubHostname) { - - var apiRequest = xhr(root, accessToken, githubHostname); - - repo.loadAs = loadAs; // (type, hash) -> value, hash - repo.saveAs = saveAs; // (type, value) -> hash, value - repo.listRefs = listRefs; // (filter='') -> [ refs ] - repo.readRef = readRef; // (ref) -> hash - repo.updateRef = updateRef; // (ref, hash) -> hash - repo.deleteRef = deleteRef // (ref) -> null - repo.createTree = createTree; // (entries) -> hash, tree - repo.hasHash = hasHash; - - function loadAs(type, hash, callback) { - if (!callback) return loadAs.bind(repo, type, hash); - // Github doesn't like empty trees, but we know them already. - if (type === "tree" && hash === emptyTree) return callback(null, {}, hash); - apiRequest("GET", "/repos/:root/git/" + type + "s/" + hash, onValue); - - function onValue(err, xhr, result) { - if (err) return callback(err); - if (xhr.status < 200 || xhr.status >= 500) { - return callback(new Error("Invalid HTTP response: " + xhr.statusCode + " " + result.message)); - } - if (xhr.status >= 300 && xhr.status < 500) return callback(); - var body; - try { body = decoders[type].call(repo, result); } - catch (err) { return callback(err); } - if (hashAs(type, body) !== hash) { - if (fixDate(type, body, hash)) console.log(type + " repaired", hash); - else console.warn("Unable to repair " + type, hash); - } - typeCache[hash] = type; - return callback(null, body, hash); - } - } - - function hasHash(hash, callback) { - if (!callback) return hasHash.bind(repo, hash); - var type = typeCache[hash]; - var types = type ? [type] : ["tag", "commit", "tree", "blob"]; - start(); - function start() { - type = types.pop(); - if (!type) return callback(null, false); - apiRequest("GET", "/repos/:root/git/" + type + "s/" + hash, onValue); - } - - function onValue(err, xhr, result) { - if (err) return callback(err); - if (xhr.status < 200 || xhr.status >= 500) { - return callback(new Error("Invalid HTTP response: " + xhr.statusCode + " " + result.message)); - } - if (xhr.status >= 300 && xhr.status < 500) return start(); - typeCache[hash] = type; - callback(null, true); - } - } - - function saveAs(type, body, callback) { - if (!callback) return saveAs.bind(repo, type, body); - var hash; - try { - hash = hashAs(type, body); + let apiRequest = xhr(root, accessToken, githubHostname); + repo.loadAs = loadAs; + repo.saveAs = saveAs; + repo.listRefs = listRefs; + repo.readRef = readRef; + repo.updateRef = updateRef; + repo.deleteRef = deleteRef; + repo.createTree = createTree; + repo.hasHash = hasHash; + function loadAs(type, hash, callback) { + if (!callback) + return loadAs.bind(repo, type, hash); + if (type === "tree" && hash === emptyTree) + return callback(null, {}, hash); + apiRequest("GET", "/repos/:root/git/" + type + "s/" + hash, onValue); + function onValue(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status < 200 || xhr.status >= 500) { + return callback(new Error("Invalid HTTP response: " + xhr.statusCode + " " + result.message)); + } + if (xhr.status >= 300 && xhr.status < 500) + return callback(); + let body; + try { + body = decoders[type].call(repo, result); + } + catch (err) { + return callback(err); + } + if (hashAs(type, body) !== hash) { + if (fixDate(type, body, hash)) + console.log(type + " repaired", hash); + else + console.warn("Unable to repair " + type, hash); + } + typeCache[hash] = type; + return callback(null, body, hash); + } } - catch (err) { - return callback(err); + function hasHash(hash, callback) { + if (!callback) + return hasHash.bind(repo, hash); + let type = typeCache[hash]; + let types = type ? [type] : ["tag", "commit", "tree", "blob"]; + start(); + function start() { + type = types.pop(); + if (!type) + return callback(null, false); + apiRequest("GET", "/repos/:root/git/" + type + "s/" + hash, onValue); + } + function onValue(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status < 200 || xhr.status >= 500) { + return callback(new Error("Invalid HTTP response: " + xhr.statusCode + " " + result.message)); + } + if (xhr.status >= 300 && xhr.status < 500) + return start(); + typeCache[hash] = type; + callback(null, true); + } } - typeCache[hash] = type; - repo.hasHash(hash, function (err, has) { - if (err) return callback(err); - if (has) return callback(null, hash, body); - - var request; - try { - request = encoders[type](body); - } - catch (err) { - return callback(err); - } - - // Github doesn't allow creating empty trees. - if (type === "tree" && request.tree.length === 0) { - return callback(null, emptyTree, body); - } - return apiRequest("POST", "/repos/:root/git/" + type + "s", request, onWrite); - - }); - - function onWrite(err, xhr, result) { - if (err) return callback(err); - if (xhr.status < 200 || xhr.status >= 300) { - return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); - } - return callback(null, result.sha, body); + function saveAs(type, body, callback) { + if (!callback) + return saveAs.bind(repo, type, body); + let hash; + try { + hash = hashAs(type, body); + } + catch (err) { + return callback(err); + } + typeCache[hash] = type; + repo.hasHash(hash, function (err, has) { + if (err) + return callback(err); + if (has) + return callback(null, hash, body); + let request; + try { + request = encoders[type](body); + } + catch (err) { + return callback(err); + } + if (type === "tree" && request.tree.length === 0) { + return callback(null, emptyTree, body); + } + return apiRequest("POST", "/repos/:root/git/" + type + "s", request, onWrite); + }); + function onWrite(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, result.sha, body); + } } - } - - // Create a tree with optional deep paths and create new blobs. - // Entries is an array of {mode, path, hash|content} - // Also deltas can be specified by setting entries.base to the hash of a tree - // in delta mode, entries can be removed by specifying just {path} - function createTree(entries, callback) { - if (!callback) return createTree.bind(repo, entries); - var toDelete = entries.base && entries.filter(function (entry) { - return !entry.mode; - }).map(function (entry) { - return entry.path; - }); - var toCreate = entries.filter(function (entry) { - return bodec.isBinary(entry.content); - }); - - if (!toCreate.length) return next(); - var done = false; - var left = entries.length; - toCreate.forEach(function (entry) { - repo.saveAs("blob", entry.content, function (err, hash) { - if (done) return; - if (err) { - done = true; - return callback(err); + function createTree(entries, callback) { + if (!callback) + return createTree.bind(repo, entries); + let toDelete = entries.base && entries.filter(function (entry) { + return !entry.mode; + }).map(function (entry) { + return entry.path; + }); + let toCreate = entries.filter(function (entry) { + return bodec.isBinary(entry.content); + }); + if (!toCreate.length) + return next(); + let done = false; + let left = entries.length; + toCreate.forEach(function (entry) { + repo.saveAs("blob", entry.content, function (err, hash) { + if (done) + return; + if (err) { + done = true; + return callback(err); + } + delete entry.content; + entry.hash = hash; + left--; + if (!left) + next(); + }); + }); + function next(err) { + if (err) + return callback(err); + if (toDelete && toDelete.length) { + return slowUpdateTree(entries, toDelete, callback); + } + return fastUpdateTree(entries, callback); } - delete entry.content; - entry.hash = hash; - left--; - if (!left) next(); - }); - }); - - function next(err) { - if (err) return callback(err); - if (toDelete && toDelete.length) { - return slowUpdateTree(entries, toDelete, callback); - } - return fastUpdateTree(entries, callback); } - } - - function fastUpdateTree(entries, callback) { - var request = { tree: entries.map(mapTreeEntry) }; - if (entries.base) request.base_tree = entries.base; - - apiRequest("POST", "/repos/:root/git/trees", request, onWrite); - - function onWrite(err, xhr, result) { - if (err) return callback(err); - if (xhr.status < 200 || xhr.status >= 300) { - return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); - } - return callback(null, result.sha, decoders.tree(result)); + function fastUpdateTree(entries, callback) { + let request = { tree: entries.map(mapTreeEntry) }; + if (entries.base) + request.base_tree = entries.base; + apiRequest("POST", "/repos/:root/git/trees", request, onWrite); + function onWrite(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, result.sha, decoders.tree(result)); + } } - } - - // Github doesn't support deleting entries via the createTree API, so we - // need to manually create those affected trees and modify the request. - function slowUpdateTree(entries, toDelete, callback) { - callback = singleCall(callback); - var root = entries.base; - - var left = 0; - - // Calculate trees that need to be re-built and save any provided content. - var parents = {}; - toDelete.forEach(function (path) { - var parentPath = path.substr(0, path.lastIndexOf("/")); - var parent = parents[parentPath] || (parents[parentPath] = { - add: {}, del: [] - }); - var name = path.substr(path.lastIndexOf("/") + 1); - parent.del.push(name); - }); - var other = entries.filter(function (entry) { - if (!entry.mode) return false; - var parentPath = entry.path.substr(0, entry.path.lastIndexOf("/")); - var parent = parents[parentPath]; - if (!parent) return true; - var name = entry.path.substr(entry.path.lastIndexOf("/") + 1); - if (entry.hash) { - parent.add[name] = { - mode: entry.mode, - hash: entry.hash - }; - return false; - } - left++; - repo.saveAs("blob", entry.content, function(err, hash) { - if (err) return callback(err); - parent.add[name] = { - mode: entry.mode, - hash: hash - }; - if (!--left) onParents(); - }); - return false; - }); - if (!left) onParents(); - - function onParents() { - Object.keys(parents).forEach(function (parentPath) { - left++; - // TODO: remove this dependency on pathToEntry - repo.pathToEntry(root, parentPath, function (err, entry) { - if (err) return callback(err); - var tree = entry.tree; - var commands = parents[parentPath]; - commands.del.forEach(function (name) { - delete tree[name]; - }); - for (var name in commands.add) { - tree[name] = commands.add[name]; - } - repo.saveAs("tree", tree, function (err, hash, tree) { - if (err) return callback(err); - other.push({ - path: parentPath, - hash: hash, - mode: modes.tree + function slowUpdateTree(entries, toDelete, callback) { + callback = singleCall(callback); + let root = entries.base; + let left = 0; + let parents = {}; + toDelete.forEach(function (path) { + let parentPath = path.substr(0, path.lastIndexOf("/")); + let parent = parents[parentPath] || (parents[parentPath] = { + add: {}, del: [] }); - if (!--left) { - other.base = entries.base; - if (other.length === 1 && other[0].path === "") { - return callback(null, hash, tree); - } - fastUpdateTree(other, callback); + let name = path.substr(path.lastIndexOf("/") + 1); + parent.del.push(name); + }); + let other = entries.filter(function (entry) { + if (!entry.mode) + return false; + let parentPath = entry.path.substr(0, entry.path.lastIndexOf("/")); + let parent = parents[parentPath]; + if (!parent) + return true; + let name = entry.path.substr(entry.path.lastIndexOf("/") + 1); + if (entry.hash) { + parent.add[name] = { + mode: entry.mode, + hash: entry.hash + }; + return false; } - }); + left++; + repo.saveAs("blob", entry.content, function (err, hash) { + if (err) + return callback(err); + parent.add[name] = { + mode: entry.mode, + hash: hash + }; + if (!--left) + onParents(); + }); + return false; }); - }); - } - } - - - function readRef(ref, callback) { - if (!callback) return readRef.bind(repo, ref); - if (ref === "HEAD") ref = "refs/heads/master"; - if (!(/^refs\//).test(ref)) { - return callback(new TypeError("Invalid ref: " + ref)); - } - return apiRequest("GET", "/repos/:root/git/" + ref, onRef); - - function onRef(err, xhr, result) { - if (err) return callback(err); - if (xhr.status === 404) return callback(); - if (xhr.status < 200 || xhr.status >= 300) { - return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); - } - return callback(null, result.object.sha); - } - } - - function deleteRef(ref, callback) { - if (!callback) return deleteRef.bind(repo, ref); - if (ref === "HEAD") ref = "refs/heads/master"; - if (!(/^refs\//).test(ref)) { - return callback(new TypeError("Invalid ref: " + ref)); + if (!left) + onParents(); + function onParents() { + Object.keys(parents).forEach(function (parentPath) { + left++; + repo.pathToEntry(root, parentPath, function (err, entry) { + if (err) + return callback(err); + let tree = entry.tree; + let commands = parents[parentPath]; + commands.del.forEach(function (name) { + delete tree[name]; + }); + for (let name in commands.add) { + tree[name] = commands.add[name]; + } + repo.saveAs("tree", tree, function (err, hash, tree) { + if (err) + return callback(err); + other.push({ + path: parentPath, + hash: hash, + mode: modes.tree + }); + if (!--left) { + other.base = entries.base; + if (other.length === 1 && other[0].path === "") { + return callback(null, hash, tree); + } + fastUpdateTree(other, callback); + } + }); + }); + }); + } } - return apiRequest("DELETE", "/repos/:root/git/" + ref, onRef); - - function onRef(err, xhr, result) { - if (err) return callback(err); - if (xhr.status === 404) return callback(); - if (xhr.status < 200 || xhr.status >= 300) { - return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); - } - return callback(null, null); + function readRef(ref, callback) { + if (!callback) + return readRef.bind(repo, ref); + if (ref === "HEAD") + ref = "refs/heads/master"; + if (!(/^refs\//).test(ref)) { + return callback(new TypeError("Invalid ref: " + ref)); + } + return apiRequest("GET", "/repos/:root/git/" + ref, onRef); + function onRef(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status === 404) + return callback(); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, result.object.sha); + } } - } - - function listRefs(filter, callback) { - if (!callback) return listRefs.bind(repo, filter); - filter = filter ? '/' + filter : ''; - return apiRequest("GET", "/repos/:root/git/refs" + filter, onResult); - - function onResult(err, xhr, result) { - if (err) return callback(err); - if (xhr.status === 404) return callback(); - if (xhr.status < 200 || xhr.status >= 300) { - return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); - } - - callback(null, result.map(function(entry) { return entry.ref })); + function deleteRef(ref, callback) { + if (!callback) + return deleteRef.bind(repo, ref); + if (ref === "HEAD") + ref = "refs/heads/master"; + if (!(/^refs\//).test(ref)) { + return callback(new TypeError("Invalid ref: " + ref)); + } + return apiRequest("DELETE", "/repos/:root/git/" + ref, onRef); + function onRef(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status === 404) + return callback(); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, null); + } } - } - - function updateRef(ref, hash, callback, force) { - if (!callback) return updateRef.bind(repo, ref, hash); - if (!(/^refs\//).test(ref)) { - return callback(new Error("Invalid ref: " + ref)); + function listRefs(filter, callback) { + if (!callback) + return listRefs.bind(repo, filter); + filter = filter ? '/' + filter : ''; + return apiRequest("GET", "/repos/:root/git/refs" + filter, onResult); + function onResult(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status === 404) + return callback(); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + callback(null, result.map(function (entry) { return entry.ref; })); + } } - return apiRequest("PATCH", "/repos/:root/git/" + ref, { - sha: hash, - force: !!force - }, onResult); - - function onResult(err, xhr, result) { - if (err) return callback(err); - if (xhr.status === 422 && result.message === "Reference does not exist") { - return apiRequest("POST", "/repos/:root/git/refs", { - ref: ref, - sha: hash + function updateRef(ref, hash, callback, force) { + if (!callback) + return updateRef.bind(repo, ref, hash); + if (!(/^refs\//).test(ref)) { + return callback(new Error("Invalid ref: " + ref)); + } + return apiRequest("PATCH", "/repos/:root/git/" + ref, { + sha: hash, + force: !!force }, onResult); - } - if (xhr.status < 200 || xhr.status >= 300) { - return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); - } - if (err) return callback(err); - callback(null, hash); + function onResult(err, xhr, result) { + if (err) + return callback(err); + if (xhr.status === 422 && result.message === "Reference does not exist") { + return apiRequest("POST", "/repos/:root/git/refs", { + ref: ref, + sha: hash + }, onResult); + } + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + if (err) + return callback(err); + callback(null, hash); + } } - - } - }; - -// GitHub has a nasty habit of stripping whitespace from messages and losing -// the timezone. This information is required to make our hashes match up, so -// we guess it by mutating the value till the hash matches. -// If we're unable to match, we will just force the hash when saving to the cache. function fixDate(type, value, hash) { - if (type !== "commit" && type !== "tag") return; - // Add up to 3 extra newlines and try all 30-minutes timezone offsets. - var clone = JSON.parse(JSON.stringify(value)); - for (var x = 0; x < 3; x++) { - for (var i = -720; i < 720; i += 30) { - if (type === "commit") { - clone.author.date.offset = i; - clone.committer.date.offset = i; - } - else if (type === "tag") { - clone.tagger.date.offset = i; - } - if (hash !== hashAs(type, clone)) continue; - // Apply the changes and return. - value.message = clone.message; - if (type === "commit") { - value.author.date.offset = clone.author.date.offset; - value.committer.date.offset = clone.committer.date.offset; - } - else if (type === "tag") { - value.tagger.date.offset = clone.tagger.date.offset; - } - return true; + if (type !== "commit" && type !== "tag") + return; + let clone = JSON.parse(JSON.stringify(value)); + for (let x = 0; x < 3; x++) { + for (let i = -720; i < 720; i += 30) { + if (type === "commit") { + clone.author.date.offset = i; + clone.committer.date.offset = i; + } + else if (type === "tag") { + clone.tagger.date.offset = i; + } + if (hash !== hashAs(type, clone)) + continue; + value.message = clone.message; + if (type === "commit") { + value.author.date.offset = clone.author.date.offset; + value.committer.date.offset = clone.committer.date.offset; + } + else if (type === "tag") { + value.tagger.date.offset = clone.tagger.date.offset; + } + return true; + } + clone.message += "\n"; } - clone.message += "\n"; - } - return false; + return false; } - function mapTreeEntry(entry) { - if (!entry.mode) throw new TypeError("Invalid entry"); - var mode = modeToString(entry.mode); - var item = { - path: entry.path, - mode: mode, - type: modeToType[mode] - }; - // Magic hash for empty file since github rejects empty contents. - if (entry.content === "") entry.hash = emptyBlob; - - if (entry.hash) item.sha = entry.hash; - else item.content = entry.content; - return item; + if (!entry.mode) + throw new TypeError("Invalid entry"); + let mode = modeToString(entry.mode); + let item = { + path: entry.path, + mode: mode, + type: modeToType[mode] + }; + if (entry.content === "") + entry.hash = emptyBlob; + if (entry.hash) + item.sha = entry.hash; + else + item.content = entry.content; + return item; } - function encodeCommit(commit) { - var out = {}; - out.message = commit.message; - out.tree = commit.tree; - if (commit.parents) out.parents = commit.parents; - else if (commit.parent) out.parents = [commit.parent]; - else commit.parents = []; - if (commit.author) out.author = encodePerson(commit.author); - if (commit.committer) out.committer = encodePerson(commit.committer); - return out; + let out = {}; + out.message = commit.message; + out.tree = commit.tree; + if (commit.parents) + out.parents = commit.parents; + else if (commit.parent) + out.parents = [commit.parent]; + else + commit.parents = []; + if (commit.author) + out.author = encodePerson(commit.author); + if (commit.committer) + out.committer = encodePerson(commit.committer); + return out; } - function encodeTag(tag) { - return { - tag: tag.tag, - message: tag.message, - object: tag.object, - tagger: encodePerson(tag.tagger) - }; + return { + tag: tag.tag, + message: tag.message, + object: tag.object, + tagger: encodePerson(tag.tagger) + }; } - function encodePerson(person) { - return { - name: person.name, - email: person.email, - date: encodeDate(person.date) - }; + return { + name: person.name, + email: person.email, + date: encodeDate(person.date) + }; } - function encodeTree(tree) { - return { - tree: Object.keys(tree).map(function (name) { - var entry = tree[name]; - var mode = modeToString(entry.mode); - return { - path: name, - mode: mode, - type: modeToType[mode], - sha: entry.hash - }; - }) - }; + return { + tree: Object.keys(tree).map(function (name) { + let entry = tree[name]; + let mode = modeToString(entry.mode); + return { + path: name, + mode: mode, + type: modeToType[mode], + sha: entry.hash + }; + }) + }; } - function encodeBlob(blob) { - if (typeof blob === "string") return { - content: bodec.encodeUtf8(blob), - encoding: "utf-8" - }; - if (bodec.isBinary(blob)) return { - content: bodec.toBase64(blob), - encoding: "base64" - }; - throw new TypeError("Invalid blob type, must be binary or string"); + if (typeof blob === "string") + return { + content: bodec.encodeUtf8(blob), + encoding: "utf-8" + }; + else if (bodec.isBinary(blob)) + return { + content: bodec.toBase64(blob), + encoding: "base64" + }; + throw new TypeError("Invalid blob type, must be binary or string"); } - function modeToString(mode) { - var string = mode.toString(8); - // Github likes all modes to be 6 chars long - if (string.length === 5) string = "0" + string; - return string; + let string = mode.toString(8); + if (string.length === 5) + string = "0" + string; + return string; } - function decodeCommit(result) { - return { - tree: result.tree.sha, - parents: result.parents.map(function (object) { - return object.sha; - }), - author: pickPerson(result.author), - committer: pickPerson(result.committer), - message: result.message - }; + return { + tree: result.tree.sha, + parents: result.parents.map(function (object) { + return object.sha; + }), + author: pickPerson(result.author), + committer: pickPerson(result.committer), + message: result.message + }; } - function decodeTag(result) { - return { - object: result.object.sha, - type: result.object.type, - tag: result.tag, - tagger: pickPerson(result.tagger), - message: result.message - }; + return { + object: result.object.sha, + type: result.object.type, + tag: result.tag, + tagger: pickPerson(result.tagger), + message: result.message + }; } - function decodeTree(result) { - var tree = {}; - result.tree.forEach(function (entry) { - tree[entry.path] = { - mode: parseInt(entry.mode, 8), - hash: entry.sha - }; - }); - return tree; + let tree = {}; + result.tree.forEach(function (entry) { + tree[entry.path] = { + mode: parseInt(entry.mode, 8), + hash: entry.sha + }; + }); + return tree; } - function decodeBlob(result) { - if (result.encoding === 'base64') { - return bodec.fromBase64(result.content.replace(/\n/g, '')); - } - if (result.encoding === 'utf-8') { - return bodec.fromUtf8(result.content); - } - throw new Error("Unknown blob encoding: " + result.encoding); + if (result.encoding === 'base64') { + return bodec.fromBase64(result.content.replace(/\n/g, '')); + } + else if (result.encoding === 'utf-8') { + return bodec.fromUtf8(result.content); + } + throw new Error("Unknown blob encoding: " + result.encoding); } - function pickPerson(person) { - return { - name: person.name, - email: person.email, - date: parseDate(person.date) - }; + return { + name: person.name, + email: person.email, + date: parseDate(person.date) + }; } - -function parseDate(string) { - // TODO: test this once GitHub adds timezone information - var match = string.match(/(-?)([0-9]{2}):([0-9]{2})$/); - var date = new Date(string); - var timezoneOffset = 0; - if (match) { - timezoneOffset = (match[1] === "-" ? 1 : -1) * ( - parseInt(match[2], 10) * 60 + parseInt(match[3], 10) - ); - } - return { - seconds: date.valueOf() / 1000, - offset: timezoneOffset - }; +function parseDate(str) { + let match = str.match(/(-?)([0-9]{2}):([0-9]{2})$/); + let date = new Date(str); + let timezoneOffset = 0; + if (match) { + timezoneOffset = (match[1] === "-" ? 1 : -1) * (parseInt(match[2], 10) * 60 + parseInt(match[3], 10)); + } + return { + seconds: date.valueOf() / 1000, + offset: timezoneOffset + }; } - function encodeDate(date) { - var seconds = date.seconds - (date.offset) * 60; - var d = new Date(seconds * 1000); - var string = d.toISOString(); - var neg = "+"; - var offset = date.offset; - if (offset <= 0) offset = -offset; - else neg = "-"; - var hours = (date.offset / 60)|0; - var minutes = date.offset % 60; - string = string.substring(0, string.lastIndexOf(".")) + - neg + twoDigit(hours) + ":" + twoDigit(minutes); - return string; + let seconds = date.seconds - (date.offset) * 60; + let d = new Date(seconds * 1000); + let string = d.toISOString(); + let neg = "+"; + let offset = date.offset; + if (offset <= 0) + offset = -offset; + else + neg = "-"; + let hours = (date.offset / 60) | 0; + let minutes = date.offset % 60; + string = string.substring(0, string.lastIndexOf(".")) + + neg + twoDigit(hours) + ":" + twoDigit(minutes); + return string; } - -// Run some quick unit tests to make sure date encoding works. [ - { offset: 300, seconds: 1401938626 }, - { offset: 400, seconds: 1401938626 } + { offset: 300, seconds: 1401938626 }, + { offset: 400, seconds: 1401938626 } ].forEach(function (date) { - var verify = parseDate(encodeDate(date)); - if (verify.seconds !== date.seconds || verify.offset !== date.offset) { - throw new Error("Verification failure testing date encoding"); - } + let verify = parseDate(encodeDate(date)); + if (verify.seconds !== date.seconds || verify.offset !== date.offset) { + throw new Error("Verification failure testing date encoding"); + } }); - function twoDigit(num) { - if (num < 10) return "0" + num; - return "" + num; + if (num < 10) + return "0" + num; + return "" + num; } - function singleCall(callback) { - var done = false; - return function () { - if (done) return console.warn("Discarding extra callback"); - done = true; - return callback.apply(this, arguments); - }; + let done = false; + return function () { + if (done) + return console.warn("Discarding extra callback"); + done = true; + return callback.apply(this, arguments); + }; } - function hashAs(type, body) { - var buffer = frame({type: type, body: body}); - return sha1(buffer); + let buffer = frame({ type: type, body: body }); + return sha1(buffer); } diff --git a/mixins/github-db.ts b/mixins/github-db.ts new file mode 100644 index 0000000..126229a --- /dev/null +++ b/mixins/github-db.ts @@ -0,0 +1,576 @@ +"use strict"; + +var modes = require('js-git/lib/modes'); +var xhr = require('../lib/xhr'); +var bodec = require('bodec'); +var sha1 = require('git-sha1'); +var frame = require('js-git/lib/object-codec').frame; + +const modeToType = { + "040000": "tree", + "100644": "blob", // normal file + "100755": "blob", // executable file + "120000": "blob", // symlink + "160000": "commit" // gitlink +}; + +const encoders = { + commit: encodeCommit, + tag: encodeTag, + tree: encodeTree, + blob: encodeBlob +}; + +const decoders = { + commit: decodeCommit, + tag: decodeTag, + tree: decodeTree, + blob: decodeBlob, +}; + +let typeCache = {}; + +// Precompute hashes for empty blob and empty tree since github won't +const empty = bodec.create(0); +const emptyBlob = sha1(frame({ type: "blob", body: empty })); +const emptyTree = sha1(frame({ type: "tree", body: empty })); + +// Implement the js-git object interface using github APIs +module.exports = function(repo, root, accessToken, githubHostname) { + + let apiRequest = xhr(root, accessToken, githubHostname); + + repo.loadAs = loadAs; // (type, hash) -> value, hash + repo.saveAs = saveAs; // (type, value) -> hash, value + repo.listRefs = listRefs; // (filter='') -> [ refs ] + repo.readRef = readRef; // (ref) -> hash + repo.updateRef = updateRef; // (ref, hash) -> hash + repo.deleteRef = deleteRef // (ref) -> null + repo.createTree = createTree; // (entries) -> hash, tree + repo.hasHash = hasHash; + + function loadAs(type, hash, callback) { + if (!callback) return loadAs.bind(repo, type, hash); + // Github doesn't like empty trees, but we know them already. + if (type === "tree" && hash === emptyTree) return callback(null, {}, hash); + apiRequest("GET", "/repos/:root/git/" + type + "s/" + hash, onValue); + + function onValue(err, xhr, result) { + if (err) return callback(err); + if (xhr.status < 200 || xhr.status >= 500) { + return callback(new Error("Invalid HTTP response: " + xhr.statusCode + " " + result.message)); + } + if (xhr.status >= 300 && xhr.status < 500) return callback(); + let body; + try { body = decoders[type].call(repo, result); } + catch (err) { return callback(err); } + if (hashAs(type, body) !== hash) { + if (fixDate(type, body, hash)) console.log(type + " repaired", hash); + else console.warn("Unable to repair " + type, hash); + } + typeCache[hash] = type; + return callback(null, body, hash); + } + } + + function hasHash(hash, callback) { + if (!callback) return hasHash.bind(repo, hash); + let type = typeCache[hash]; + let types = type ? [type] : ["tag", "commit", "tree", "blob"]; + start(); + function start() { + type = types.pop(); + if (!type) return callback(null, false); + apiRequest("GET", "/repos/:root/git/" + type + "s/" + hash, onValue); + } + + function onValue(err, xhr, result) { + if (err) return callback(err); + if (xhr.status < 200 || xhr.status >= 500) { + return callback(new Error("Invalid HTTP response: " + xhr.statusCode + " " + result.message)); + } + if (xhr.status >= 300 && xhr.status < 500) return start(); + typeCache[hash] = type; + callback(null, true); + } + } + + function saveAs(type, body, callback) { + if (!callback) return saveAs.bind(repo, type, body); + let hash; + try { + hash = hashAs(type, body); + } + catch (err) { + return callback(err); + } + typeCache[hash] = type; + repo.hasHash(hash, function(err, has) { + if (err) return callback(err); + if (has) return callback(null, hash, body); + + let request; + try { + request = encoders[type](body); + } + catch (err) { + return callback(err); + } + + // Github doesn't allow creating empty trees. + if (type === "tree" && request.tree.length === 0) { + return callback(null, emptyTree, body); + } + return apiRequest("POST", "/repos/:root/git/" + type + "s", request, onWrite); + + }); + + function onWrite(err, xhr, result) { + if (err) return callback(err); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, result.sha, body); + } + } + + // Create a tree with optional deep paths and create new blobs. + // Entries is an array of {mode, path, hash|content} + // Also deltas can be specified by setting entries.base to the hash of a tree + // in delta mode, entries can be removed by specifying just {path} + function createTree(entries, callback): any { + if (!callback) return createTree.bind(repo, entries); + let toDelete = entries.base && entries.filter(function(entry) { + return !entry.mode; + }).map(function(entry) { + return entry.path; + }); + let toCreate = entries.filter(function(entry) { + return bodec.isBinary(entry.content); + }); + + if (!toCreate.length) return next(); + let done = false; + let left = entries.length; + toCreate.forEach(function(entry) { + repo.saveAs("blob", entry.content, function(err, hash) { + if (done) return; + if (err) { + done = true; + return callback(err); + } + delete entry.content; + entry.hash = hash; + left--; + if (!left) next(); + }); + }); + + function next(err?) { + if (err) return callback(err); + if (toDelete && toDelete.length) { + return slowUpdateTree(entries, toDelete, callback); + } + return fastUpdateTree(entries, callback); + } + } + + function fastUpdateTree(entries, callback) { + let request: any = { tree: entries.map(mapTreeEntry) }; + if (entries.base) request.base_tree = entries.base; + + apiRequest("POST", "/repos/:root/git/trees", request, onWrite); + + function onWrite(err, xhr, result) { + if (err) return callback(err); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, result.sha, decoders.tree(result)); + } + } + + // Github doesn't support deleting entries via the createTree API, so we + // need to manually create those affected trees and modify the request. + function slowUpdateTree(entries, toDelete, callback) { + callback = singleCall(callback); + let root = entries.base; + + let left = 0; + + // Calculate trees that need to be re-built and save any provided content. + let parents = {}; + toDelete.forEach(function(path) { + let parentPath = path.substr(0, path.lastIndexOf("/")); + let parent = parents[parentPath] || (parents[parentPath] = { + add: {}, del: [] + }); + let name = path.substr(path.lastIndexOf("/") + 1); + parent.del.push(name); + }); + let other = entries.filter(function(entry) { + if (!entry.mode) return false; + let parentPath = entry.path.substr(0, entry.path.lastIndexOf("/")); + let parent = parents[parentPath]; + if (!parent) return true; + let name = entry.path.substr(entry.path.lastIndexOf("/") + 1); + if (entry.hash) { + parent.add[name] = { + mode: entry.mode, + hash: entry.hash + }; + return false; + } + left++; + repo.saveAs("blob", entry.content, function(err, hash) { + if (err) return callback(err); + parent.add[name] = { + mode: entry.mode, + hash: hash + }; + if (!--left) onParents(); + }); + return false; + }); + if (!left) onParents(); + + function onParents() { + Object.keys(parents).forEach(function(parentPath) { + left++; + // TODO: remove this dependency on pathToEntry + repo.pathToEntry(root, parentPath, function(err, entry) { + if (err) return callback(err); + let tree = entry.tree; + let commands = parents[parentPath]; + commands.del.forEach(function(name) { + delete tree[name]; + }); + for (let name in commands.add) { + tree[name] = commands.add[name]; + } + repo.saveAs("tree", tree, function(err, hash, tree) { + if (err) return callback(err); + other.push({ + path: parentPath, + hash: hash, + mode: modes.tree + }); + if (!--left) { + other.base = entries.base; + if (other.length === 1 && other[0].path === "") { + return callback(null, hash, tree); + } + fastUpdateTree(other, callback); + } + }); + }); + }); + } + } + + + function readRef(ref, callback) { + if (!callback) return readRef.bind(repo, ref); + if (ref === "HEAD") ref = "refs/heads/master"; + if (!(/^refs\//).test(ref)) { + return callback(new TypeError("Invalid ref: " + ref)); + } + return apiRequest("GET", "/repos/:root/git/" + ref, onRef); + + function onRef(err, xhr, result) { + if (err) return callback(err); + if (xhr.status === 404) return callback(); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, result.object.sha); + } + } + + function deleteRef(ref, callback) { + if (!callback) return deleteRef.bind(repo, ref); + if (ref === "HEAD") ref = "refs/heads/master"; + if (!(/^refs\//).test(ref)) { + return callback(new TypeError("Invalid ref: " + ref)); + } + return apiRequest("DELETE", "/repos/:root/git/" + ref, onRef); + + function onRef(err, xhr, result) { + if (err) return callback(err); + if (xhr.status === 404) return callback(); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + return callback(null, null); + } + } + + function listRefs(filter: string, callback) { + if (!callback) return listRefs.bind(repo, filter); + filter = filter ? '/' + filter : ''; + return apiRequest("GET", "/repos/:root/git/refs" + filter, onResult); + + function onResult(err, xhr, result) { + if (err) return callback(err); + if (xhr.status === 404) return callback(); + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + + callback(null, result.map(function(entry) { return entry.ref })); + } + } + + function updateRef(ref, hash, callback, force) { + if (!callback) return updateRef.bind(repo, ref, hash); + if (!(/^refs\//).test(ref)) { + return callback(new Error("Invalid ref: " + ref)); + } + return apiRequest("PATCH", "/repos/:root/git/" + ref, { + sha: hash, + force: !!force + }, onResult); + + function onResult(err, xhr, result) { + if (err) return callback(err); + if (xhr.status === 422 && result.message === "Reference does not exist") { + return apiRequest("POST", "/repos/:root/git/refs", { + ref: ref, + sha: hash + }, onResult); + } + if (xhr.status < 200 || xhr.status >= 300) { + return callback(new Error("Invalid HTTP response: " + xhr.status + " " + result.message)); + } + if (err) return callback(err); + callback(null, hash); + } + + } + +}; + +// GitHub has a nasty habit of stripping whitespace from messages and losing +// the timezone. This information is required to make our hashes match up, so +// we guess it by mutating the value till the hash matches. +// If we're unable to match, we will just force the hash when saving to the cache. +function fixDate(type, value, hash) { + if (type !== "commit" && type !== "tag") return; + // Add up to 3 extra newlines and try all 30-minutes timezone offsets. + let clone = JSON.parse(JSON.stringify(value)); + for (let x = 0; x < 3; x++) { + for (let i = -720; i < 720; i += 30) { + if (type === "commit") { + clone.author.date.offset = i; + clone.committer.date.offset = i; + } + else if (type === "tag") { + clone.tagger.date.offset = i; + } + if (hash !== hashAs(type, clone)) continue; + // Apply the changes and return. + value.message = clone.message; + if (type === "commit") { + value.author.date.offset = clone.author.date.offset; + value.committer.date.offset = clone.committer.date.offset; + } + else if (type === "tag") { + value.tagger.date.offset = clone.tagger.date.offset; + } + return true; + } + clone.message += "\n"; + } + return false; +} + +function mapTreeEntry(entry) { + if (!entry.mode) throw new TypeError("Invalid entry"); + let mode = modeToString(entry.mode); + let item: any = { + path: entry.path, + mode: mode, + type: modeToType[mode] + }; + // Magic hash for empty file since github rejects empty contents. + if (entry.content === "") entry.hash = emptyBlob; + + if (entry.hash) item.sha = entry.hash; + else item.content = entry.content; + return item; +} + +function encodeCommit(commit) { + let out: any = {}; + out.message = commit.message; + out.tree = commit.tree; + if (commit.parents) out.parents = commit.parents; + else if (commit.parent) out.parents = [commit.parent]; + else commit.parents = []; + if (commit.author) out.author = encodePerson(commit.author); + if (commit.committer) out.committer = encodePerson(commit.committer); + return out; +} + +function encodeTag(tag) { + return { + tag: tag.tag, + message: tag.message, + object: tag.object, + tagger: encodePerson(tag.tagger) + }; +} + +function encodePerson(person) { + return { + name: person.name, + email: person.email, + date: encodeDate(person.date) + }; +} + +function encodeTree(tree) { + return { + tree: Object.keys(tree).map(function(name) { + let entry = tree[name]; + let mode = modeToString(entry.mode); + return { + path: name, + mode: mode, + type: modeToType[mode], + sha: entry.hash + }; + }) + }; +} + +function encodeBlob(blob) { + if (typeof blob === "string") return { + content: bodec.encodeUtf8(blob), + encoding: "utf-8" + }; + else if (bodec.isBinary(blob)) return { + content: bodec.toBase64(blob), + encoding: "base64" + }; + throw new TypeError("Invalid blob type, must be binary or string"); +} + +function modeToString(mode) { + let string = mode.toString(8); + // Github likes all modes to be 6 chars long + if (string.length === 5) string = "0" + string; + return string; +} + +function decodeCommit(result: any) { + return { + tree: result.tree.sha, + parents: result.parents.map(function(object) { + return object.sha; + }), + author: pickPerson(result.author), + committer: pickPerson(result.committer), + message: result.message + }; +} + +function decodeTag(result: any) { + return { + object: result.object.sha, + type: result.object.type, + tag: result.tag, + tagger: pickPerson(result.tagger), + message: result.message + }; +} + +function decodeTree(result) { + let tree = {}; + result.tree.forEach(function(entry) { + tree[entry.path] = { + mode: parseInt(entry.mode, 8), + hash: entry.sha + }; + }); + return tree; +} + +function decodeBlob(result) { + if (result.encoding === 'base64') { + return bodec.fromBase64(result.content.replace(/\n/g, '')); + } + else if (result.encoding === 'utf-8') { + return bodec.fromUtf8(result.content); + } + throw new Error("Unknown blob encoding: " + result.encoding); +} + +function pickPerson(person) { + return { + name: person.name, + email: person.email, + date: parseDate(person.date) + }; +} + +function parseDate(str: string) { + // TODO: test this once GitHub adds timezone information + let match = str.match(/(-?)([0-9]{2}):([0-9]{2})$/); + let date = new Date(str); + let timezoneOffset = 0; + if (match) { + timezoneOffset = (match[1] === "-" ? 1 : -1) * ( + parseInt(match[2], 10) * 60 + parseInt(match[3], 10) + ); + } + return { + seconds: date.valueOf() / 1000, + offset: timezoneOffset + }; +} + +function encodeDate(date) { + let seconds = date.seconds - (date.offset) * 60; + let d = new Date(seconds * 1000); + let string = d.toISOString(); + let neg = "+"; + let offset = date.offset; + if (offset <= 0) offset = -offset; + else neg = "-"; + let hours = (date.offset / 60) | 0; + let minutes = date.offset % 60; + string = string.substring(0, string.lastIndexOf(".")) + + neg + twoDigit(hours) + ":" + twoDigit(minutes); + return string; +} + +// Run some quick unit tests to make sure date encoding works. +[ + { offset: 300, seconds: 1401938626 }, + { offset: 400, seconds: 1401938626 } +].forEach(function(date) { + let verify = parseDate(encodeDate(date)); + if (verify.seconds !== date.seconds || verify.offset !== date.offset) { + throw new Error("Verification failure testing date encoding"); + } +}); + +function twoDigit(num: number) { + if (num < 10) return "0" + num; + return "" + num; +} + +function singleCall(callback) { + let done = false; + return function() { + if (done) return console.warn("Discarding extra callback"); + done = true; + return callback.apply(this, arguments); + }; +} + +function hashAs(type, body) { + let buffer = frame({ type: type, body: body }); + return sha1(buffer); +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..aca0aa7 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,31 @@ +{ + "compilerOptions": { + "target": "es6", + "module": "commonjs", + "moduleResolution": "node", + "isolatedModules": false, + "jsx": "react", + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "declaration": false, + "noImplicitAny": false, + "removeComments": true, + "noLib": false, + "preserveConstEnums": true, + "suppressImplicitAnyIndexErrors": true + }, + "filesGlob": [ + "**/*.ts", + "**/*.tsx", + "!node_modules/**" + ], + "files": [ + "examples/readme-caps.ts", + "lib/xhr-node.ts", + "lib/xhr.ts", + "mixins/github-db.ts", + "typings/node/node.d.ts", + "typings/tsd.d.ts" + ], + "exclude": [] +} diff --git a/tsd.json b/tsd.json new file mode 100644 index 0000000..101f631 --- /dev/null +++ b/tsd.json @@ -0,0 +1,12 @@ +{ + "version": "v4", + "repo": "borisyankov/DefinitelyTyped", + "ref": "master", + "path": "typings", + "bundle": "typings/tsd.d.ts", + "installed": { + "node/node.d.ts": { + "commit": "3191f6e0088eee07c4d8fd24e4d27a40a60d9eb9" + } + } +}