svn commit: r1909780 - /manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java

[kwright]

Author: kwright
Date: Fri May 12 18:43:59 2023
New Revision: 1909780

URL: http://svn.apache.org/viewvc?rev=1909780&view=rev
Log:
CONNECTORS-1746: Add support for smarter analysis

Modified:
    
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java

Modified: 
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
URL: 
http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java?rev=1909780&r1=1909779&r2=1909780&view=diff
==============================================================================
--- 
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
 (original)
+++ 
manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/database/DBInterfacePostgreSQL.java
 Fri May 12 18:43:59 2023
@@ -1374,7 +1374,7 @@ public class DBInterfacePostgreSQL exten
       try
       {
         String eventDatum = statsAnalyzePrefix+tableName;
-        // Time to reindex this table!
+        // Time to analyze this table!
         analyzeTableInternal(tableName);
         // Now, clear out the data
         writeDatum(eventDatum,0);
@@ -1574,17 +1574,46 @@ public class DBInterfacePostgreSQL exten
         {
           String eventDatum = statsAnalyzePrefix+tableName;
           int oldEventCount = readDatum(eventDatum);
-          oldEventCount += ts.getEventCount();
-          if (oldEventCount >= analyzeThreshold)
+          int tsEventCount = ts.getEventCount();
+          oldEventCount += tsEventCount;
+          long currentTime = System.currentTimeMillis();
+          // If property "analyzeatstart" is set, then analyze this table when 
the job starts
+          boolean analyzeAtStart = 
lockManager.getSharedConfiguration().getBooleanProperty("org.apache.manifoldcf.db.postgres.analyzeatstart",
 false);
+          if (analyzeAtStart && ts.isFirstAnalyze())
           {
-            // Time to reindex this table!
             analyzeTableInternal(tableName);
-            // Now, clear out the data
             writeDatum(eventDatum,0);
+            // Set the firstAnalyze flag to false
+            ts.setFirstAnalyze(false);
+          }
+          else if (oldEventCount >= analyzeThreshold)
+          {
+            // If property "analyzeratethreshold" is set, then analyze this 
table only when events per second drops below the threshold
+            int analyzeRateThreshold = 
lockManager.getSharedConfiguration().getIntProperty("org.apache.manifoldcf.db.postgres.analyzeratethreshold",
 0);
+            boolean skipAnalyze = false;
+            long previousTime = ts.getPreviousTime();
+            if (analyzeRateThreshold > 0 && previousTime > 0L)
+            {
+              long elapsedTime = currentTime - previousTime;
+              if (elapsedTime > 0)
+              {
+                int eventRate = (int)(tsEventCount * 1000L / elapsedTime);
+                if (eventRate >= analyzeRateThreshold)
+                  skipAnalyze = true;
+              }
+            }
+            if (!skipAnalyze)
+            {
+              // Time to analyze this table!
+              analyzeTableInternal(tableName);
+              // Now, clear out the data
+              writeDatum(eventDatum,0);
+            }
           }
           else
             writeDatum(eventDatum,oldEventCount);
           ts.reset();
+          ts.setPreviousTime(currentTime);
         }
         finally
         {
@@ -1605,6 +1634,8 @@ public class DBInterfacePostgreSQL exten
   protected static class TableStatistics
   {
     protected int eventCount = 0;
+    protected boolean firstAnalyze = true;
+    protected long timeMilliseconds = 0L;
     
     public TableStatistics()
     {
@@ -1624,6 +1655,26 @@ public class DBInterfacePostgreSQL exten
     {
       return eventCount;
     }
+
+    public boolean isFirstAnalyze()
+    {
+      return this.firstAnalyze;
+    }
+
+    public void setFirstAnalyze(boolean firstAnalyze)
+    {
+      this.firstAnalyze = firstAnalyze;
+    }
+
+    public long getPreviousTime()
+    {
+      return this.timeMilliseconds;
+    }
+
+    public void setPreviousTime(long timeMilliseconds)
+    {
+      this.timeMilliseconds = timeMilliseconds;
+    }
   }
   
 }