Merge pull request #2812 from chaoss/dev

Release Ready

README.md

@@ -1,4 +1,4 @@
-# Augur NEW Release v0.70.0
+# Augur NEW Release v0.71.0
 
 Augur is primarily a data engineering tool that makes it possible for data scientists to gather open source software community data. Less data carpentry for everyone else! 
 The primary way of looking at Augur data is through [8Knot](https://github.com/oss-aspen/8knot) ... A public instance of 8Knot is available at https://metrix.chaoss.io ... That is tied to a public instance of Augur at https://ai.chaoss.io 
@@ -10,7 +10,7 @@ The primary way of looking at Augur data is through [8Knot](https://github.com/o
 ## NEW RELEASE ALERT!
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.70.0  
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.71.0  
 
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.

augur/api/routes/config.py

@@ -8,6 +8,7 @@
 
 # Disable the requirement for SSL by setting env["AUGUR_DEV"] = True
 from augur.application.config import get_development_flag
+from augur.application.db.lib import get_session
 from augur.application.db.models import Config
 from augur.application.config import AugurConfig
 from augur.application.db.session import DatabaseSession
@@ -45,7 +46,7 @@ def update_config():
 
     update_dict = request.get_json()
 
-    with DatabaseSession(logger, engine=current_app.engine) as session:
+    with get_session() as session:
 
         for section, data in update_dict.items():
 

augur/api/routes/dei.py

@@ -92,7 +92,7 @@ def core_task_success_util_gen(repo_git):
     deiHook = CollectionRequest("core",primary_enabled_phases)
     deiHook.repo_list = [repo_url]
 
-    singleRoutine = AugurTaskRoutine(session,[deiHook])
+    singleRoutine = AugurTaskRoutine(logger, session,[deiHook])
     singleRoutine.start_data_collection()
     #start_block_of_repos(logger, session, [repo_url], primary_enabled_phases, "new")
 

augur/api/view/augur_view.py

@@ -48,6 +48,7 @@ def internal_server_error(error):
         traceback.print_tb(error.__traceback__, file=errout)
         # traceback.print_exception(error, file=errout)
         stacktrace = errout.getvalue()
+        stacktrace += f"\n{type(error).__name__}: {str(error)}"
         errout.close()
     except Exception as e:
         logger.error(e)

augur/application/cli/backend.py

@@ -100,6 +100,7 @@ def start(ctx, disable_collection, development, port):
         create_collection_status_records.si().apply_async()
         time.sleep(3)
 
+        #put contributor breadth back in. Not sure why it was commented out
         contributor_breadth_model.si().apply_async()
 
         # start cloning repos when augur starts
@@ -164,15 +165,15 @@ def determine_worker_processes(ratio,maximum):
         process_list.append(subprocess.Popen(scheduling_worker.split(" ")))
         sleep_time += 6
 
-        #60% of estimate, Maximum value of 45
-        core_num_processes = determine_worker_processes(.6, 45)
+        #60% of estimate, Maximum value of 45 : Reduced because it can be lower
+        core_num_processes = determine_worker_processes(.15, 10)
         logger.info(f"Starting core worker processes with concurrency={core_num_processes}")
         core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={core_num_processes} -n core:{uuid.uuid4().hex}@%h"
         process_list.append(subprocess.Popen(core_worker.split(" ")))
         sleep_time += 6
 
         #20% of estimate, Maximum value of 25
-        secondary_num_processes = determine_worker_processes(.25, 45)
+        secondary_num_processes = determine_worker_processes(.70, 60)
         logger.info(f"Starting secondary worker processes with concurrency={secondary_num_processes}")
         secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={secondary_num_processes} -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
         process_list.append(subprocess.Popen(secondary_worker.split(" ")))
@@ -317,7 +318,7 @@ def assign_orphan_repos_to_default_user(session):
     repos = session.execute_sql(query).fetchall()
 
     for repo in repos:
-        UserRepo.insert(session,repo[0],1)
+        UserRepo.insert(session, repo[0],1)
 
 
 @cli.command('export-env')

augur/application/cli/collection.py

@@ -124,15 +124,15 @@ def determine_worker_processes(ratio,maximum):
     process_list.append(subprocess.Popen(scheduling_worker.split(" ")))
     sleep_time += 6
 
-    #60% of estimate, Maximum value of 45
-    core_num_processes = determine_worker_processes(.6, 45)
+    #60% of estimate, Maximum value of 45: Reduced because not needed
+    core_num_processes = determine_worker_processes(.15, 10)
     logger.info(f"Starting core worker processes with concurrency={core_num_processes}")
     core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={core_num_processes} -n core:{uuid.uuid4().hex}@%h"
     process_list.append(subprocess.Popen(core_worker.split(" ")))
     sleep_time += 6
 
     #20% of estimate, Maximum value of 25
-    secondary_num_processes = determine_worker_processes(.25, 45)
+    secondary_num_processes = determine_worker_processes(.70, 60)
     logger.info(f"Starting secondary worker processes with concurrency={secondary_num_processes}")
     secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency={secondary_num_processes} -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
     process_list.append(subprocess.Popen(secondary_worker.split(" ")))
@@ -301,4 +301,4 @@ def assign_orphan_repos_to_default_user(session):
     repos = session.execute_sql(query).fetchall()
 
     for repo in repos:
-        UserRepo.insert(session,repo[0],1)
+        UserRepo.insert(session, repo[0],1)

augur/application/cli/tasks.py

@@ -36,8 +36,8 @@ def start():
     secondary_worker_process = None
 
     scheduling_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=1 -n scheduling:{uuid.uuid4().hex}@%h -Q scheduling"
-    core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=45 -n core:{uuid.uuid4().hex}@%h"
-    secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=45 -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
+    core_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=20 -n core:{uuid.uuid4().hex}@%h"
+    secondary_worker = f"celery -A augur.tasks.init.celery_app.celery_app worker -l info --concurrency=60 -n secondary:{uuid.uuid4().hex}@%h -Q secondary"
 
     scheduling_worker_process = subprocess.Popen(scheduling_worker.split(" "))
     core_worker_process = subprocess.Popen(core_worker.split(" "))

augur/application/config.py

@@ -288,6 +288,7 @@ def add_or_update_settings(self, settings: List[dict]):
             query = self.session.query(Config).filter(and_(Config.section_name == setting["section_name"],Config.setting_name == setting["setting_name"]) )
 
             if execute_session_query(query, 'first') is None:
+                # TODO: Update to use bulk insert dicts so config doesn't require database session
                 self.session.insert_data(setting,Config, ["section_name", "setting_name"])
             else:
                 #If setting exists. use raw update to not increase autoincrement