Merge remote-tracking branch 'upstream/master' into allow-multiple-keywords-in-search

Author: carols10cents

.buildpacks

@@ -1,4 +1,3 @@
-https://github.com/Starkast/heroku-buildpack-cmake#a243c67
 https://github.com/emk/heroku-buildpack-rust#578d630
 https://codon-buildpacks.s3.amazonaws.com/buildpacks/heroku/emberjs.tgz
 https://github.com/heroku/heroku-buildpack-nginx.git#fbc49cd

.travis.yml

@@ -1,6 +1,6 @@
 language: rust
 sudo: required
-dist: trusty
+dist: xenial
 
 # Ignore this branch per bors-ng documentation
 branches:
@@ -24,8 +24,11 @@ env:
     # on community-submitted PRs
     - PERCY_TOKEN=0d8707a02b19aebbec79bb0bf302b8d2fa95edb33169cfe41b084289596670b1
     - PERCY_PROJECT=crates-io/crates.io
+    - PGPORT=5433
 
 install:
+  - sudo cp /etc/postgresql/10/main/pg_hba.conf /etc/postgresql/11/main/pg_hba.conf
+  - sudo systemctl restart postgresql@11-main
   - script/ci/cargo-clean-on-new-rustc-version.sh
   - cargo install --force diesel_cli --vers `cat .diesel_version` --no-default-features --features postgres && export PATH=$HOME/.cargo/bin:$PATH
 
@@ -34,15 +37,11 @@ before_script:
 
 addons:
   chrome: stable
-  postgresql: "9.5"
+  postgresql: "11"
   apt:
-    sources:
-      - kalakris-cmake
     packages:
-      - cmake
-      - libcurl4-openssl-dev
-      - libelf-dev
-      - libdw-dev
+      - postgresql-11
+      - postgresql-client-11
 
 matrix:
   fast_finish: true

Cargo.lock

@@ -29,13 +29,12 @@ rustdoc-args = [
 
 [dependencies]
 cargo-registry-s3 = { path = "src/s3", version = "0.2.0" }
-old_semver = { path = "src/old_semver", version = "0.1.0" }
-rand = "0.3"
-git2 = "0.6.4"
+rand = "0.6"
+git2 = "0.8.0"
 flate2 = "1.0"
 semver = { version = "0.9", git = "https://github.com/steveklabnik/semver.git", features = ["diesel", "serde"] }
 url = "1.2.1"
-tar = "0.4.13"
+tar = "0.4.16"
 base64 = "0.9"
 
 openssl = "0.10.13"
@@ -49,26 +48,24 @@ dotenv = "0.11.0"
 toml = "0.4"
 diesel = { version = "1.4.0", features = ["postgres", "serde_json", "chrono", "r2d2"] }
 diesel_full_text_search = "1.0.0"
-diesel_ltree = "0.1.3"
+swirl = { git = "https://github.com/sgrif/swirl.git", rev = "de5d8bb" }
 serde_json = "1.0.0"
-serde_derive = "1.0.0"
-serde = "1.0.0"
+serde = { version = "1.0.0", features = ["derive"] }
 chrono = { version = "0.4.0", features = ["serde"] }
-comrak = { version = "0.2.3", default-features = false }
-ammonia = "1.0.0"
+comrak = { version = "0.4.0", default-features = false }
+ammonia = "2.0.0"
 docopt = "0.8.1"
-itertools = "0.7.0"
 scheduled-thread-pool = "0.2.0"
 derive_deref = "1.0.0"
 reqwest = "0.9.1"
 tempdir = "0.3.7"
 collecting-hashmap = "0.2"
 parking_lot = "0.7.1"
-jemallocator = { version = "0.1.8", features = ['unprefixed_malloc_on_supported_platforms'] }
+jemallocator = { version = "0.1.8", features = ['unprefixed_malloc_on_supported_platforms', 'profiling'] }
 jemalloc-ctl = "0.2.0"
 
-lettre = {git = "https://github.com/lettre/lettre", version = "0.9"}
-lettre_email = {git = "https://github.com/lettre/lettre", version = "0.9"}
+lettre = "0.9"
+lettre_email = "0.9"
 
 conduit = "0.8"
 conduit-conditional-get = "0.8"
@@ -79,6 +76,7 @@ conduit-router = "0.8"
 conduit-static = "0.8"
 conduit-git-http-backend = "0.8"
 civet = "0.9"
+conduit-hyper = "0.1.3"
 
 [dev-dependencies]
 conduit-test = "0.8"
@@ -87,7 +85,6 @@ hyper-tls = "0.3"
 futures = "0.1"
 lazy_static = "1.0"
 tokio-core = "0.1"
-tokio-service = "0.1"
 
 [build-dependencies]
 dotenv = "0.11"

Cargo.toml

@@ -1,2 +1,2 @@
 web: bin/diesel migration run && bin/start-nginx ./target/release/server
-worker: ./target/release/update-downloads daemon 300
+background_worker: ./target/release/background-worker

Procfile

@@ -1,6 +1,7 @@
 # crates.io
 
 [![Build Status](https://travis-ci.org/rust-lang/crates.io.svg?branch=master)](https://travis-ci.org/rust-lang/crates.io)
+[![What's Deployed](https://img.shields.io/badge/whatsdeployed-prod-green.svg)](https://whatsdeployed.io/s-9IG)
 
 Source code for the default [Cargo](http://doc.crates.io) registry. Viewable
 online at [crates.io](https://crates.io).
@@ -15,7 +16,7 @@ of the following ways:
 
 - [File a new issue](https://github.com/rust-lang/crates.io/issues/new)
 - Email [[email protected]](mailto:[email protected])
-- Chat on irc.mozilla.org in the [#rust-infra](https://kiwiirc.com/client/irc.mozilla.org:+6667/#rust-infra) channel
+- Chat on ops > #crates-io channel on https://discord.gg/rust-lang
 
 A volunteer will get back to you as soon as possible.
 

README.md

@@ -0,0 +1 @@
+VERSION=1.32.0

RustConfig

@@ -78,10 +78,20 @@ export default Component.extend({
 
         let myData = window.google.visualization.arrayToDataTable(data);
 
-        let fmt = new window.google.visualization.DateFormat({
+        let dateFmt = new window.google.visualization.DateFormat({
             pattern: 'LLL d, yyyy',
         });
-        fmt.format(myData, 0);
+        dateFmt.format(myData, 0);
+
+        // Create a formatter to use for daily download numbers
+        let numberFormatWhole = new window.google.visualization.NumberFormat({
+            pattern: '#,##0',
+        });
+
+        // Create a formatter to use for 7-day average numbers
+        let numberFormatDecimal = new window.google.visualization.NumberFormat({
+            pattern: '#,##0.0',
+        });
 
         // use a DataView to calculate an x-day moving average
         let days = 7;
@@ -101,7 +111,7 @@ export default Component.extend({
                 let avg = total / days;
                 return {
                     v: avg,
-                    f: avg.toFixed(2),
+                    f: numberFormatDecimal.formatValue(avg),
                 };
             };
         };
@@ -113,6 +123,8 @@ export default Component.extend({
         // is at the end, but in the UI we want it at the top of the chart legend.
 
         range(headers.length - 1, 0, -1).forEach((dataCol, i) => {
+            // Set the number format for the colum in the data table.
+            numberFormatWhole.format(myData, dataCol);
             columns.push(dataCol); // add the column itself
             columns.push({
                 // add a 'calculated' column, the moving average

app/components/download-graph.js

@@ -62,7 +62,7 @@ export default Route.extend({
                     .get('versions')
                     .then(versions => {
                         const latestStableVersion = versions.find(version => {
-                            if (!isUnstableVersion(version.get('num'))) {
+                            if (!isUnstableVersion(version.get('num')) && !version.get('yanked')) {
                                 return version;
                             }
                         });

app/routes/crate/version.js

@@ -1,5 +1,5 @@
 import Route from '@ember/routing/route';
-import ajax from 'ember-fetch/ajax';
+import fetch from 'fetch';
 import { serializeQueryParams } from 'ember-fetch/mixins/adapter-fetch';
 
 /**
@@ -19,8 +19,9 @@ export default Route.extend({
     async beforeModel(transition) {
         try {
             let queryParams = serializeQueryParams(transition.queryParams);
-            let d = await ajax(`/authorize?${queryParams}`);
-            let item = JSON.stringify({ ok: true, data: d });
+            let resp = await fetch(`/authorize?${queryParams}`);
+            let json = await resp.json();
+            let item = JSON.stringify({ ok: resp.ok, data: json });
             if (window.opener) {
                 window.opener.github_response = item;
             }

app/routes/github-authorize.js

@@ -53,15 +53,15 @@ export default Route.extend({
             if (!response) {
                 return;
             }
-            if (!response.ok) {
-                this.flashMessages.show('Failed to log in');
-                return;
-            }
+
             let { data } = response;
-            if (data.errors) {
+            if (data && data.errors) {
                 let error = `Failed to log in: ${data.errors[0].detail}`;
                 this.flashMessages.show(error);
                 return;
+            } else if (!response.ok) {
+                this.flashMessages.show('Failed to log in');
+                return;
             }
 
             let user = this.store.push(this.store.normalize('user', data.user));

app/routes/login.js

@@ -20,7 +20,10 @@
                 placeholder="Click or press 'S' to search..."
                 value={{searchQuery}}
                 oninput={{action (mut searchQuery) value="target.value"}}
+                autocorrect="off"
+                autocapitalize="off"
                 autofocus="autofocus"
+                spellcheck="false"
                 required
                 data-test-search-input>
         <label for="cargo-desktop-search">Search</label>

app/templates/application.hbs

@@ -233,7 +233,7 @@
                     <h3>Dev-Dependencies</h3>
                     <ul>
                         {{#each currentDevDependencies as |dep|}}
-                            {{link-to-dep dep=dep}}
+                            {{link-to-dep tagName="li" dep=dep}}
                         {{/each}}
                     </ul>
                 </div>

app/templates/crate/version.hbs

@@ -1,7 +1,7 @@
 FROM rust:latest
 
 RUN apt-get update \
-    && apt-get install -y postgresql cmake \
+    && apt-get install -y postgresql \
     && rm -rf /var/lib/apt/lists/* \
     && cargo install diesel_cli --no-default-features --features postgres
 

backend.Dockerfile

@@ -67,7 +67,7 @@ http {
 		location ~ ^/api/v./crates/new$ {
 			proxy_pass http://app_server;
 
-			limit_req zone=publish burst=10 nodelay;
+			limit_req zone=publish burst=30 nodelay;
 			limit_req_status 429;
 		}
 	}

config/nginx.conf.erb

@@ -3,5 +3,5 @@
 [print_schema]
 file = "src/schema.rs"
 with_docs = true
-import_types = ["diesel::sql_types::*", "diesel_full_text_search::{TsVector as Tsvector}", "diesel_ltree::Ltree"]
+import_types = ["diesel::sql_types::*", "diesel_full_text_search::{TsVector as Tsvector}"]
 patch_file = "src/schema.patch"

diesel.toml

@@ -37,7 +37,7 @@ services:
     build:
       context: .
       dockerfile: frontend.Dockerfile
-    entrypoint: npm run start --proxy http://backend:8888
+    entrypoint: npm run start -- --proxy http://backend:8888
     links:
       - backend
     ports:

docker-compose.yml

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # If the backend is started before postgres is ready, the migrations will fail
-until diesel migration run; do
+until diesel migration run --locked-schema; do
   echo "Migrations failed, retrying in 5 seconds..."
   sleep 5
 done

docker_entrypoint.sh

@@ -12,12 +12,12 @@ The server does the following things:
 3. Reads values from environment variables to configure a new instance of `cargo_registry::App`
 4. Adds middleware to the app by calling `cargo_registry::middleware`
 5. Syncs the categories defined in *src/categories.toml* with the categories in the database
-6. Starts a [civet][] `Server` that uses the `cargo_registry::App` instance
+6. Starts either a [conduit] or a [hyper] server that uses the `cargo_registry::App` instance
 7. Tells Nginx on Heroku that the application is ready to receive requests, if running on Heroku
-8. Blocks forever (or until the process is killed) waiting to receive messages on a channel that no
-   messages are ever sent to, in order to outive the civet `Server` threads
+8. Blocks forever (or until the process is killed)
 
 [civet]: https://crates.io/crates/civet
+[hyper]: https://crates.io/crates/hyper
 
 ## Routes
 

docs/BACKEND.md

@@ -145,7 +145,6 @@ In order to run the backend, you will need to have installed:
 
 - [Rust](https://www.rust-lang.org/en-US/) stable >= 1.16.0 and cargo, which comes with Rust
 - [Postgres](https://www.postgresql.org/) >= 9.5
-- [CMake](https://cmake.org/download/) >= 3.6.1
 - [OpenSSL](https://www.openssl.org/) >= 1.0.2k
 - [diesel_cli](http://diesel.rs/guides/getting-started/) >= 1.2.0
 
@@ -230,15 +229,6 @@ by typing `\q`) without any errors to connect to your running Postgres server.
 > we'll help fix the problem and and will add the solution to these
 > instructions!
 
-##### CMake
-
-- All platforms: CMake [has binary distributions
-  available](https://cmake.org/download/)
-- macOS: you can also install with homebrew by using `brew install cmake`
-- Linux: you should be able to use the distribution repositories by doing `sudo
-  apt-get install cmake` (Ubuntu) or `sudo dnf install cmake` (Fedora) or
-  whatever is appropriate for your distribution.
-
 ##### OpenSSL
 
 - Windows: [Win32 OpenSSL Installation

docs/CONTRIBUTING.md

@@ -1,7 +1,9 @@
 FROM node:8.6-alpine
 
 WORKDIR /app
-COPY package.json /app
+COPY package.json package-lock.json /app/
+
+RUN npm install
 
 COPY . /app
 

frontend.Dockerfile

@@ -0,0 +1 @@
+DROP TABLE background_jobs;

migrations/2018-05-03-150523_create_jobs/down.sql

@@ -0,0 +1,8 @@
+CREATE TABLE background_jobs (
+  id BIGSERIAL PRIMARY KEY,
+  job_type TEXT NOT NULL,
+  data JSONB NOT NULL,
+  retries INTEGER NOT NULL DEFAULT 0,
+  last_retry TIMESTAMP NOT NULL DEFAULT '1970-01-01',
+  created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);

migrations/2018-05-03-150523_create_jobs/up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE versions DROP CONSTRAINT fk_versions_published_by;
+
+ALTER TABLE versions DROP COLUMN published_by;

migrations/2018-11-01-223239_add_published_by_to_versions/down.sql

@@ -0,0 +1,7 @@
+ALTER TABLE versions
+ADD COLUMN published_by integer;
+
+ALTER TABLE versions
+ADD CONSTRAINT "fk_versions_published_by"
+FOREIGN KEY (published_by)
+REFERENCES users(id);

migrations/2018-11-01-223239_add_published_by_to_versions/up.sql

@@ -0,0 +1 @@
+DROP TABLE versions_published_by;

migrations/2019-02-12-200540_track_version_published_by_email/down.sql

@@ -0,0 +1,4 @@
+CREATE TABLE versions_published_by (
+    version_id INTEGER NOT NULL PRIMARY KEY REFERENCES versions(id) ON DELETE CASCADE,
+    email VARCHAR NOT NULL
+);

migrations/2019-02-12-200540_track_version_published_by_email/up.sql

@@ -0,0 +1,6 @@
+DROP MATERIALIZED VIEW recent_crate_downloads;
+CREATE MATERIALIZED VIEW recent_crate_downloads (crate_id, downloads) AS
+  SELECT crate_id, SUM(downloads) FROM crate_downloads
+    WHERE date > date(CURRENT_TIMESTAMP - INTERVAL '90 days')
+    GROUP BY crate_id;
+CREATE UNIQUE INDEX recent_crate_downloads_crate_id ON recent_crate_downloads (crate_id);

migrations/2019-03-22-162022_define_recent_crate_downloads_in_terms_of_version_downloads/down.sql

@@ -0,0 +1,8 @@
+DROP MATERIALIZED VIEW recent_crate_downloads;
+CREATE MATERIALIZED VIEW recent_crate_downloads (crate_id, downloads) AS
+  SELECT crate_id, SUM(version_downloads.downloads) FROM version_downloads
+    INNER JOIN versions
+      ON version_downloads.version_id = versions.id
+    WHERE version_downloads.date > date(CURRENT_TIMESTAMP - INTERVAL '90 days')
+    GROUP BY crate_id;
+CREATE UNIQUE INDEX recent_crate_downloads_crate_id ON recent_crate_downloads (crate_id);