Add more aggressive rate limiting for publishing new crates

RustConfig

@@ -1 +1 @@
-VERSION=1.32.0
+VERSION=1.33.0

migrations/2019-03-18-233900_create_publish_limit_buckets/down.sql

@@ -0,0 +1 @@
+DROP TABLE publish_limit_buckets;

migrations/2019-03-18-233900_create_publish_limit_buckets/up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE publish_limit_buckets(
+  user_id INTEGER PRIMARY KEY NOT NULL REFERENCES users,
+  tokens INTEGER NOT NULL,
+  last_refill TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
+);

migrations/2019-04-04-192902_create_publish_rate_overrides/down.sql

@@ -0,0 +1 @@
+DROP TABLE publish_rate_overrides;

migrations/2019-04-04-192902_create_publish_rate_overrides/up.sql

@@ -0,0 +1,4 @@
+CREATE TABLE publish_rate_overrides (
+  user_id INTEGER PRIMARY KEY REFERENCES users,
+  burst INTEGER NOT NULL
+);

src/bin/update-downloads.rs

@@ -107,7 +107,7 @@ mod test {
             name: "foo",
             ..Default::default()
         }
-        .create_or_update(conn, None, user_id)
+        .create_or_update(conn, None, user_id, None)
         .unwrap();
         let version = NewVersion::new(
             krate.id,

src/config.rs

@@ -1,3 +1,4 @@
+use crate::publish_rate_limit::PublishRateLimit;
 use crate::{env, uploaders::Uploader, Env, Replica};
 use std::path::PathBuf;
 use url::Url;
@@ -16,6 +17,7 @@ pub struct Config {
     pub max_unpack_size: u64,
     pub mirror: Replica,
     pub api_protocol: String,
+    pub publish_rate_limit: PublishRateLimit,
 }
 
 impl Default for Config {
@@ -132,6 +134,7 @@ impl Default for Config {
             max_unpack_size: 512 * 1024 * 1024, // 512 MB max when decompressed
             mirror,
             api_protocol,
+            publish_rate_limit: Default::default(),
         }
     }
 }

src/controllers/krate/publish.rs

@@ -84,7 +84,12 @@ pub fn publish(req: &mut dyn Request) -> CargoResult<Response> {
         };
 
         let license_file = new_crate.license_file.as_ref().map(|s| &**s);
-        let krate = persist.create_or_update(&conn, license_file, user.id)?;
+        let krate = persist.create_or_update(
+            &conn,
+            license_file,
+            user.id,
+            Some(&app.config.publish_rate_limit),
+        )?;
 
         let owners = krate.owners(&conn)?;
         if user.rights(req.app(), &owners)? < Rights::Publish {

src/lib.rs

@@ -37,8 +37,10 @@ pub mod email;
 pub mod git;
 pub mod github;
 pub mod middleware;
+mod publish_rate_limit;
 pub mod render;
 pub mod schema;
+mod test_util;
 pub mod uploaders;
 pub mod util;
 

src/models/category.rs

@@ -183,12 +183,11 @@ impl<'a> NewCategory<'a> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::test_util::pg_connection_no_transaction;
     use diesel::connection::SimpleConnection;
 
     fn pg_connection() -> PgConnection {
-        let database_url =
-            dotenv::var("TEST_DATABASE_URL").expect("TEST_DATABASE_URL must be set to run tests");
-        let conn = PgConnection::establish(&database_url).unwrap();
+        let conn = pg_connection_no_transaction();
         // These tests deadlock if run concurrently
         conn.batch_execute("BEGIN; LOCK categories IN ACCESS EXCLUSIVE MODE")
             .unwrap();

src/models/krate.rs

@@ -15,6 +15,7 @@ use crate::models::{
 use crate::views::{EncodableCrate, EncodableCrateLinks};
 
 use crate::models::helpers::with_count::*;
+use crate::publish_rate_limit::PublishRateLimit;
 use crate::schema::*;
 
 /// Hosts in this list are known to not be hosting documentation,
@@ -105,6 +106,7 @@ impl<'a> NewCrate<'a> {
         conn: &PgConnection,
         license_file: Option<&'a str>,
         uploader: i32,
+        rate_limit: Option<&PublishRateLimit>,
     ) -> CargoResult<Crate> {
         use diesel::update;
 
@@ -115,6 +117,9 @@ impl<'a> NewCrate<'a> {
             // To avoid race conditions, we try to insert
             // first so we know whether to add an owner
             if let Some(krate) = self.save_new_crate(conn, uploader)? {
+                if let Some(rate_limit) = rate_limit {
+                    rate_limit.check_rate_limit(uploader, conn)?;
+                }
                 return Ok(krate);
             }
 

src/models/user.rs

@@ -21,7 +21,7 @@ pub struct User {
     pub gh_id: i32,
 }
 
-#[derive(Insertable, Debug)]
+#[derive(Insertable, Debug, Default)]
 #[table_name = "users"]
 pub struct NewUser<'a> {
     pub gh_id: i32,