Skip to content

i1 #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Dec 13, 2016
Merged

i1 #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
80f4f9c
i1
ptitzler Dec 13, 2016
c3a13da
i1: address jshint warnings
ptitzler Dec 13, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 39 additions & 16 deletions index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ const path = require('path');
const SimpleDataVis = require('simple-data-vis');

// to enable debugging, set environment variable DEBUG to nps or *
const debug = require('debug')('npmwd');
const debug = require('debug')('npmjs');

const init = require('./lib/initialize.js');

Expand Down Expand Up @@ -60,7 +60,7 @@ const init = require('./lib/initialize.js');
console.log('Service is initializing...');

// initialize Cloudant repository and load defaults
init(appEnv, function(err, dataRepository, metaRepository, collector) {
init(appEnv, function(err, dataRepository, metaRepository, collector, indexer) {

//
// dataRepository - cloudant handle for data database
Expand All @@ -71,23 +71,46 @@ const init = require('./lib/initialize.js');
process.exit(1);
}

/* var year = 2015;
//var month = null;
// asynchronously build index and collect missing statistics
debug('Building index ...');
indexer.buildIndex(function(err) {
if(err) {
console.error('Index build error: ' + err);
}
else {
debug('Index was built. Identifying gaps ...');
indexer.inspectIndex(null,
function(err, todolist) {
if(err) {
console.error('Index inspection error: ' + err);
}
else {
debug('Collecting missing statistics ... ');
collector.collect(todolist,
function(err, data) {
if(err) {
console.error('Statistics collector error: ' + err);
}
else {
debug('Statistics collection results: ' + JSON.stringify(data));
}
});
}
});
}
});

collector.collectYear(['ibm-graph-client', 'couchimport', 'cf-deployment-tracker-client', 'cachemachine', 'cloudant-envoy', 'nosqlimport','nosqlimport-couchdb','nosqlimport-mongodb', 'nosqlelasticsearch'],
year,
//month,
function (err) {
if(err) {
console.log("Collection error: " + err);
}
else {
console.log("Collection done");
}

/* TODO remove
// seed statistics
collector.collectMonth('cloudant',
2015,
12,
function(err) {
console.log(err);
});

*/

var app = express();
app.use(bodyParser.urlencoded({extended: false}));
// use https://www.npmjs.com/package/express-handlebars as view engine
Expand Down Expand Up @@ -124,7 +147,7 @@ const init = require('./lib/initialize.js');
// render stats page
collector.getPackages(function(err, packages) {
if(err) {
console.log("/stats: error retrieving package list: " + err);
console.log('/stats: error retrieving package list: ' + err);
res.render('stats');
}
res.render('stats', {packages:packages});
Expand Down
65 changes: 52 additions & 13 deletions lib/collector.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ function NPMJSCollector(repository) {
NPMJSCollector.prototype.getPackages = function(callback) {

if(! this.dataRepository) {
return callback("No data repository is defined.");
return callback('No data repository is defined.');
}

var options = {
Expand Down Expand Up @@ -85,31 +85,35 @@ NPMJSCollector.prototype.collectYear = function(packages, year, callback) {
NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback) {

if(! this.dataRepository) {
return callback("No data repository is defined.");
return callback('No data repository is defined.');
}

if(! packages) {
return callback(); // nothing to do
}

// TODO
// check year/month parms


if(! Array.isArray(packages)) {
packages = [packages];
}

var dateRangeStart = new Date(year, month - 1,1).toISOString().substring(0,10);
var dateRangeEnd = new Date(year, month, 0).toISOString().substring(0,10);

debug("Collecting download statistics for " + packages.toString() + " for date range " + dateRangeStart + " and " + dateRangeEnd);
debug('Collecting download statistics for ' + packages.toString() + ' for date range ' + dateRangeStart + ' and ' + dateRangeEnd);

// submit npmjs request
// sample: https://api.npmjs.org/downloads/range/2016-01-01:2016-02-01/cloudant
this.npmjsClient.getAll({ dateRange: dateRangeStart + ":" + dateRangeEnd , packages: packages},
this.npmjsClient.getAll({ dateRange: dateRangeStart + ':' + dateRangeEnd , packages: packages},
function(err, data) {
if(err) {
return callback(err);
}

if(data.hasOwnProperty("error")) {
if(data.hasOwnProperty('error')) {
if(data.error === 'no stats for this package for this range (0008)') {
return callback(); // nothing to do; don't return an error
}
Expand All @@ -118,22 +122,22 @@ NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback
}
}

debug("Err: " + JSON.stringify(err));
debug("Data: " + JSON.stringify(data));
debug('Err: ' + JSON.stringify(err));
debug('Data: ' + JSON.stringify(data));

// iterate through packages and create a stats doc for each
var statsDocs = [];
var root = null;
var total = 0;
_.forEach(packages,
function(packageName) {
debug("Checking for download information for package " + packageName);
debug('Checking for download information for package ' + packageName);
root = null;
if(data.hasOwnProperty(packageName)) {
root = data[packageName];
}
else {
if(data.hasOwnProperty("downloads")) {
if(data.hasOwnProperty('downloads')) {
root = data;
}
}
Expand All @@ -145,10 +149,10 @@ NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback
});

statsDocs.push({
_id: root["package"] + "_" + root.start.substring(0, 7),
type: "stats",
package: root["package"],
language: "javascript",
_id: root['package'] + '_' + root.start.substring(0, 7),
type: 'stats',
package: root['package'],
language: 'javascript',
month: root.start.substring(0, 7),
total: total,
range: {
Expand All @@ -174,4 +178,39 @@ NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback
}.bind(this));
};

/**
* Collects the statistics for todolist
* @param todolist
* @param callback invoked when processing is finished; returns (err, data)
*/
NPMJSCollector.prototype.collect = function(todolist, callback) {
if(! todolist) {
// nothing to do
return callback();
}
// process TODO list
async.eachLimit(Object.keys(todolist),
1,
function(year, asyncCallbackYear) {
async.eachLimit(
Object.keys(todolist[year]),
1,
function(month, asyncCallbackMonth) {
this.collectMonth(todolist[year][month],
year,
month,
function(err, data) {
debug(data);
return asyncCallbackMonth(err);
});
}.bind(this),
function(err) {
return asyncCallbackYear(err);
});
}.bind(this),
function(err) {
return callback(err);
});
};

module.exports = NPMJSCollector;
Loading