Revision: 3455
Author: [email protected]
Date: Wed Apr 14 09:10:02 2010
Log: NEW - bug 2748: Profling is not persisted
http://trillian.sqlpower.ca/bugzilla/show_bug.cgi?id=2748
Profiling is now persisted to the server. Previously the profiles were
being added to an older profile manager.
Added BigDecimal, BigInteger, Timestamp and friends to the converter for
profiles to be persisted.
The Long fields in the JCR can now be used to store ints or longs.
Previously, and for the most case, longs store ints and
convert the value in the JCR to an int. If the numeric field is meant to be
treated as a long it will be flagged as such
since it is uncommon.
http://code.google.com/p/power-architect/source/detail?r=3455
Modified:
/trunk/regress/ca/sqlpower/architect/profile/RemoteDatabateProfileCreatorTest.java
/trunk/regress/ca/sqlpower/architect/profile/TableProfileManagerTest.java
/trunk/regress/ca/sqlpower/architect/profile/TestProfileCSV.java
/trunk/src/main/java/ca/sqlpower/architect/ArchitectProject.java
/trunk/src/main/java/ca/sqlpower/architect/enterprise/ArchitectSessionPersister.java
/trunk/src/main/java/ca/sqlpower/architect/profile/AbstractProfileResult.java
/trunk/src/main/java/ca/sqlpower/architect/profile/AbstractTableProfileCreator.java
/trunk/src/main/java/ca/sqlpower/architect/profile/ColumnProfileResult.java
/trunk/src/main/java/ca/sqlpower/architect/profile/ColumnValueCount.java
/trunk/src/main/java/ca/sqlpower/architect/profile/ProfileManagerImpl.java
/trunk/src/main/java/ca/sqlpower/architect/profile/TableProfileResult.java
/trunk/src/main/java/ca/sqlpower/architect/swingui/DataMoverPanel.java
/trunk/src/main/java/ca/sqlpower/architect/swingui/ProfileRowComponent.java
=======================================
---
/trunk/regress/ca/sqlpower/architect/profile/RemoteDatabateProfileCreatorTest.java
Mon Jun 1 07:48:14 2009
+++
/trunk/regress/ca/sqlpower/architect/profile/RemoteDatabateProfileCreatorTest.java
Wed Apr 14 09:10:02 2010
@@ -38,6 +38,8 @@
import java.util.Collection;
import junit.framework.TestCase;
+import ca.sqlpower.architect.ArchitectProject;
+import ca.sqlpower.architect.StubArchitectSession;
import ca.sqlpower.sql.DataSourceCollection;
import ca.sqlpower.sql.JDBCDataSource;
import ca.sqlpower.sql.JDBCDataSourceType;
@@ -91,7 +93,25 @@
RemoteDatabaseProfileCreator rdpc = new
RemoteDatabaseProfileCreator(new ProfileSettings());
SQLDatabase db = new SQLDatabase(ds);
SQLTable table = db.getTableByName("test_table");
+
+ final ArchitectProject project = new ArchitectProject();
+ StubArchitectSession session = new StubArchitectSession() {
+ @Override
+ public ArchitectProject getWorkspace() {
+ return project;
+ }
+ @Override
+ public void runInForeground(Runnable runner) {
+ runner.run();
+ }
+ };
+ project.setSession(session);
+ ProfileManager profileManager = new ProfileManagerImpl();
+ project.setProfileManager(profileManager);
TableProfileResult tpr = new TableProfileResult(table, new
ProfileSettings());
+ profileManager.addChild(tpr, 0);
+ project.getRootObject().addDatabase(db, 0);
+
rdpc.doProfile(tpr);
Collection<ColumnProfileResult> cprCollection =
tpr.getColumnProfileResult(table.getColumn(0));
assertEquals(1, cprCollection.size());
=======================================
---
/trunk/regress/ca/sqlpower/architect/profile/TableProfileManagerTest.java
Thu Jan 29 12:02:55 2009
+++
/trunk/regress/ca/sqlpower/architect/profile/TableProfileManagerTest.java
Wed Apr 14 09:10:02 2010
@@ -130,6 +130,7 @@
public void testAsynchCreateProfiles() throws Exception {
SQLTable garbageTable = new SQLTable();
garbageTable.setName("Not profilable");
+ mydb.addTable(garbageTable);
List<SQLTable> tables = Arrays.asList(t1, garbageTable, t2, t3);
int oldCount = pm.getResults().size();
=======================================
--- /trunk/regress/ca/sqlpower/architect/profile/TestProfileCSV.java Thu
Jan 29 12:02:55 2009
+++ /trunk/regress/ca/sqlpower/architect/profile/TestProfileCSV.java Wed
Apr 14 09:10:02 2010
@@ -64,7 +64,7 @@
rdr.readLine();
}
assertNotNull(line = rdr.readLine()); // third results line,
column t1_c4
- assertTrue("Incorrect line: " + line,
line.endsWith("\"32,345.7\",\"\""));
+ assertTrue("Incorrect line: " + line,
line.endsWith("\"32345.6789\",\"\""));
}
}
=======================================
--- /trunk/src/main/java/ca/sqlpower/architect/ArchitectProject.java Fri
Apr 9 08:41:23 2010
+++ /trunk/src/main/java/ca/sqlpower/architect/ArchitectProject.java Wed
Apr 14 09:10:02 2010
@@ -113,7 +113,7 @@
* @throws SQLObjectException
*/
public ArchitectProject() throws SQLObjectException {
- this(new SQLObjectRoot(), new OLAPRootObject(), new
KettleSettings());
+ this(new SQLObjectRoot(), new OLAPRootObject(), new
KettleSettings(), null);
SQLDatabase targetDatabase = new SQLDatabase();
targetDatabase.setPlayPenDatabase(true);
rootObject.addChild(targetDatabase, 0);
@@ -126,12 +126,21 @@
* @param rootObject
* The root object that holds all of the source databases
for the
* current project.
+ * @param olapRootObject
+ * The root object of OLAP projects. All OLAP projects will
be
+ * contained under this node.
+ * @param kettleSettings
+ * The settings to create Kettle jobs for this project.
+ * @param profileManager
+ * The default profile manager for this project. This may
be null
+ * if it is set later or the profile manager is not used.
*/
@Constructor
public ArchitectProject(
@ConstructorParameter(isProperty=ParameterType.CHILD,
propertyName="rootObject") SQLObjectRoot rootObject,
@ConstructorParameter(isProperty=ParameterType.CHILD,
propertyName="olapRootObject") OLAPRootObject olapRootObject,
- @ConstructorParameter(isProperty=ParameterType.CHILD,
propertyName="kettleSettings") KettleSettings kettleSettings)
+ @ConstructorParameter(isProperty=ParameterType.CHILD,
propertyName="kettleSettings") KettleSettings kettleSettings,
+ @ConstructorParameter(isProperty=ParameterType.CHILD,
propertyName="profileManager") ProfileManager profileManager)
throws SQLObjectException {
this.rootObject = rootObject;
rootObject.setParent(this);
@@ -141,6 +150,9 @@
projectSettings.setParent(this);
this.kettleSettings = kettleSettings;
kettleSettings.setParent(this);
+ if (profileManager != null) {
+ setProfileManager(profileManager);
+ }
setName("Architect Project");
}
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/enterprise/ArchitectSessionPersister.java
Wed Mar 17 13:09:48 2010
+++
/trunk/src/main/java/ca/sqlpower/architect/enterprise/ArchitectSessionPersister.java
Wed Apr 14 09:10:02 2010
@@ -25,6 +25,7 @@
import ca.sqlpower.architect.ArchitectProject;
import ca.sqlpower.architect.etl.kettle.KettleSettings;
import ca.sqlpower.architect.olap.OLAPRootObject;
+import ca.sqlpower.architect.profile.ProfileManagerImpl;
import ca.sqlpower.dao.PersistedSPOProperty;
import ca.sqlpower.dao.PersistedSPObject;
import ca.sqlpower.dao.SPSessionPersister;
@@ -54,6 +55,17 @@
ArchitectProject architectProject = (ArchitectProject) root;
architectProject.getRootObject().setUUID(rootObjectUUID);
persistedRootObject.setLoaded(true);
+
+ String profileManagerUUID = (String)
AbstractSPPersisterHelper.findPropertyAndRemove(
+ pso.getUUID(), "profileManager", persistedProperties);
+
+ //Null for system projects.
+ if (profileManagerUUID != null) {
+ PersistedSPObject persistedProfileManager =
AbstractSPPersisterHelper.findPersistedSPObject(
+ pso.getUUID(), ProfileManagerImpl.class.getName(),
profileManagerUUID, persistedObjects);
+
architectProject.getProfileManager().setUUID(profileManagerUUID);
+ persistedProfileManager.setLoaded(true);
+ }
String olapRootObjectUUID = (String)
AbstractSPPersisterHelper.findPropertyAndRemove(
pso.getUUID(), "olapRootObject", persistedProperties);
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/profile/AbstractProfileResult.java
Wed Feb 17 14:19:51 2010
+++
/trunk/src/main/java/ca/sqlpower/architect/profile/AbstractProfileResult.java
Wed Apr 14 09:10:02 2010
@@ -76,6 +76,15 @@
if (profiledObject == null) throw new NullPointerException("The
profiled object has to be non-null");
this.profiledObject = profiledObject;
}
+
+ public AbstractProfileResult(AbstractProfileResult<T> profileToCopy) {
+ this(profileToCopy.getProfiledObject());
+ this.createStartTime = profileToCopy.createStartTime;
+ this.createEndTime = profileToCopy.createEndTime;
+ this.ex = profileToCopy.ex;
+ //These settings come from the profile manager
+ this.settings = profileToCopy.settings;
+ }
/* (non-Javadoc)
* @see
ca.sqlpower.architect.profile.ProfileResultInterface#getProfiledObject()
@@ -132,7 +141,7 @@
/* (non-Javadoc)
* @see
ca.sqlpower.architect.profile.ProfileResultInterface#getException()
*/
- @Transient @Accessor
+ @Accessor
public Exception getException() {
return ex;
}
@@ -141,7 +150,7 @@
* If an exception is encountered while populating this profile result,
* it should be stored here for later inspection by client code.
*/
- @Transient @Mutator
+ @Mutator
public void setException(Exception ex) {
Exception oldEx = this.ex;
this.ex = ex;
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/profile/AbstractTableProfileCreator.java
Tue Mar 25 07:38:28 2008
+++
/trunk/src/main/java/ca/sqlpower/architect/profile/AbstractTableProfileCreator.java
Wed Apr 14 09:10:02 2010
@@ -21,7 +21,15 @@
import org.apache.log4j.Logger;
+import ca.sqlpower.architect.enterprise.ArchitectPersisterSuperConverter;
+import ca.sqlpower.architect.enterprise.ArchitectSessionPersister;
+import ca.sqlpower.dao.SPPersisterListener;
+import ca.sqlpower.object.SPObject;
+import ca.sqlpower.sql.DataSourceCollection;
+import ca.sqlpower.sql.PlDotIni;
+import ca.sqlpower.sql.SPDataSource;
import ca.sqlpower.util.MonitorableImpl;
+import ca.sqlpower.util.SessionNotFoundException;
/**
* Provides a template method for doProfile which alleviates the
@@ -40,35 +48,79 @@
* or without success) unless this profile population has been
cancelled,
* in which case it fires a profileCancelled event.
*/
- public final boolean doProfile(TableProfileResult tpr) {
+ public final boolean doProfile(final TableProfileResult actualTPR) {
+ final TableProfileResult tpr = new TableProfileResult(actualTPR);
+
+ //Setting the profile result UUID to match which allows the
+ //persister to update the actual tpr at the end.
+ tpr.setUUID(actualTPR.getUUID());
+
MonitorableImpl pm = (MonitorableImpl) tpr.getProgressMonitor();
try {
- tpr.fireProfileStarted();
- pm.setMessage(tpr.getProfiledObject().getName());
- if (!pm.isCancelled()) {
- pm.setStarted(true);
- pm.setFinished(false);
- tpr.setCreateStartTime(System.currentTimeMillis());
- doProfileImpl(tpr);
- }
- } catch (Exception ex) {
- tpr.setException(ex);
- logger.error("Profile failed. Saving exception:", ex);
+ tpr.begin("Profiling");
+ try {
+ tpr.fireProfileStarted();
+ pm.setMessage(tpr.getProfiledObject().getName());
+ if (!pm.isCancelled()) {
+ pm.setStarted(true);
+ pm.setFinished(false);
+ tpr.setCreateStartTime(System.currentTimeMillis());
+ doProfileImpl(tpr);
+ }
+ } catch (Exception ex) {
+ tpr.setException(ex);
+ logger.error("Profile failed. Saving exception:", ex);
+ } finally {
+ tpr.setCreateEndTime(System.currentTimeMillis());
+ pm.setProgress(pm.getProgress() + 1);
+ pm.setFinished(true);
+ // this somehow fixes a progress bar visibility issue
+ pm.setStarted(false);
+ if (pm.isCancelled()) {
+ tpr.fireProfileCancelled();
+ } else {
+ tpr.fireProfileFinished();
+ }
+ }
+ tpr.commit();
} finally {
- tpr.setCreateEndTime(System.currentTimeMillis());
- pm.setProgress(pm.getProgress() + 1);
- pm.setFinished(true);
- // this somehow fixes a progress bar visibility issue
- pm.setStarted(false);
- if (pm.isCancelled()) {
- tpr.fireProfileCancelled();
- } else {
- tpr.fireProfileFinished();
+ Runnable runner = new Runnable() {
+ public void run() {
+ //None of the profiling creates or saves any data
source information so an
+ //empty data source is used for the converter. If the
profiling stores
+ //data source information in the future we may need
the data source collection
+ //in the project.
+ DataSourceCollection<SPDataSource> dsCollection = new
PlDotIni();
+
+ SPObject root =
actualTPR.getWorkspaceContainer().getWorkspace();
+ ArchitectPersisterSuperConverter converter =
+ new ArchitectPersisterSuperConverter(dsCollection,
root);
+ ArchitectSessionPersister persister =
+ new ArchitectSessionPersister("Profiling
persister", root, converter);
+
persister.setWorkspaceContainer(actualTPR.getWorkspaceContainer());
+ SPPersisterListener eventCreator = new
SPPersisterListener(persister, converter);
+ eventCreator.persistObject(tpr,
+
actualTPR.getParent().getChildren(TableProfileResult.class).indexOf(actualTPR),
+ false);
+ }
+ };
+ try {
+ actualTPR.getRunnableDispatcher().runInForeground(runner);
+ } catch (SessionNotFoundException e) {
+ runner.run();
}
}
return !pm.isCancelled();
}
+ /**
+ * The meat of profiling the table. The {...@link TableProfileResult}
given
+ * is not connected to the table so you do not need to worry about
where the
+ * events go to.
+ *
+ * @param tpr
+ * The table to profile.
+ */
protected abstract boolean doProfileImpl(TableProfileResult tpr)
throws Exception;
}
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/profile/ColumnProfileResult.java
Thu Feb 18 13:18:27 2010
+++
/trunk/src/main/java/ca/sqlpower/architect/profile/ColumnProfileResult.java
Wed Apr 14 09:10:02 2010
@@ -64,8 +64,29 @@
public
ColumnProfileResult(@ConstructorParameter(propertyName="profiledObject")
SQLColumn profiledObject,
@ConstructorParameter(propertyName="parentResult")
TableProfileResult parentResult) {
super(profiledObject);
+ setName("New Column Profile");
this.parentResult = parentResult;
}
+
+ /**
+ * Deep-copy copy constructor.
+ */
+ public ColumnProfileResult(ColumnProfileResult cprToCopy) {
+ super(cprToCopy);
+ setName("New Column Profile");
+ this.avgLength = cprToCopy.avgLength;
+ this.avgValue = cprToCopy.avgValue;
+ this.distinctValueCount = cprToCopy.distinctValueCount;
+ this.maxLength = cprToCopy.maxLength;
+ this.minLength = cprToCopy.minLength;
+ this.minValue = cprToCopy.minValue;
+ this.nullCount = cprToCopy.nullCount;
+ this.parentResult = cprToCopy.getParentResult();
+ for (int i = 0; i < cprToCopy.getValueCount().size(); i++) {
+ ColumnValueCount cvc = cprToCopy.getValueCount().get(i);
+ this.addColumnValueCount(new ColumnValueCount(cvc), i);
+ }
+ }
@Accessor
public double getAvgLength() {
@@ -204,6 +225,7 @@
private void addColumnValueCount(ColumnValueCount value, int index) {
topTen.add(value);
+ value.setParent(this);
fireChildAdded(ColumnValueCount.class, value, index);
}
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/profile/ColumnValueCount.java
Wed Feb 17 15:16:16 2010
+++
/trunk/src/main/java/ca/sqlpower/architect/profile/ColumnValueCount.java
Wed Apr 14 09:10:02 2010
@@ -59,10 +59,18 @@
public ColumnValueCount(@ConstructorParameter(propertyName="value")
Object value,
@ConstructorParameter(propertyName="count") int count,
@ConstructorParameter(propertyName="percent") double percent) {
+ setName("New Column Value Count");
this.value = value;
this.count = count;
this.percent = percent;
}
+
+ public ColumnValueCount(ColumnValueCount cvcToCopy) {
+ setName(cvcToCopy.getName());
+ this.value = cvcToCopy.value;
+ this.count = cvcToCopy.count;
+ this.percent = cvcToCopy.percent;
+ }
@Accessor
public int getCount() {
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/profile/ProfileManagerImpl.java
Wed Mar 10 08:21:50 2010
+++
/trunk/src/main/java/ca/sqlpower/architect/profile/ProfileManagerImpl.java
Wed Apr 14 09:10:02 2010
@@ -223,6 +223,9 @@
*/
private void addResults(List<TableProfileResult> newResults) {
results.addAll(newResults);
+ for (TableProfileResult tpr : newResults) {
+ tpr.setParent(this);
+ }
fireProfilesAdded(newResults);
for (TableProfileResult newResult : newResults) {
SQLTable table = newResult.getProfiledObject();
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/profile/TableProfileResult.java
Thu Feb 18 13:18:27 2010
+++
/trunk/src/main/java/ca/sqlpower/architect/profile/TableProfileResult.java
Wed Apr 14 09:10:02 2010
@@ -89,8 +89,24 @@
public
TableProfileResult(@ConstructorParameter(propertyName="profiledObject")
SQLTable profiledObject,
@ConstructorParameter(propertyName="settings") ProfileSettings
settings) {
super(profiledObject);
+ setName("New Table Profile");
setSettings(settings);
}
+
+ /**
+ * This is a deep-copy copy constructor. (ie: all of the descendants
will be
+ * copied as well). However, the progress monitor will be shared
between the
+ * results.
+ */
+ public TableProfileResult(TableProfileResult tprToCopy) {
+ super(tprToCopy);
+ setName("New Table Profile");
+ this.rowCount = tprToCopy.rowCount;
+ this.progressMonitor = tprToCopy.progressMonitor;
+ for (ColumnProfileResult cpr :
tprToCopy.getColumnProfileResults()) {
+ addColumnProfileResult(new ColumnProfileResult(cpr));
+ }
+ }
/**
* Returns the progress monitor that can be polled to track the
progress
=======================================
--- /trunk/src/main/java/ca/sqlpower/architect/swingui/DataMoverPanel.java
Wed Mar 17 13:09:48 2010
+++ /trunk/src/main/java/ca/sqlpower/architect/swingui/DataMoverPanel.java
Wed Apr 14 09:10:02 2010
@@ -170,7 +170,7 @@
try {
if (treeRoot == null) {
treeRoot = new SQLObjectRoot();
- ArchitectProject treeProject = new
ArchitectProject(treeRoot, new OLAPRootObject(), new KettleSettings());
+ ArchitectProject treeProject = new
ArchitectProject(treeRoot, new OLAPRootObject(), new KettleSettings(),
null);
treeProject.setSession(session);
treeRoot.begin("Setting up database trees in data mover
panel.");
} else {
=======================================
---
/trunk/src/main/java/ca/sqlpower/architect/swingui/ProfileRowComponent.java
Wed Aug 12 14:56:31 2009
+++
/trunk/src/main/java/ca/sqlpower/architect/swingui/ProfileRowComponent.java
Wed Apr 14 09:10:02 2010
@@ -31,6 +31,7 @@
import java.awt.event.InputEvent;
import java.awt.event.MouseAdapter;
import java.awt.event.MouseEvent;
+import java.beans.PropertyChangeEvent;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -58,6 +59,8 @@
import ca.sqlpower.architect.swingui.dbtree.DBTreeCellRenderer;
import ca.sqlpower.architect.swingui.event.SelectionEvent;
import ca.sqlpower.architect.swingui.event.SelectionListener;
+import ca.sqlpower.object.AbstractSPListener;
+import ca.sqlpower.object.SPListener;
import ca.sqlpower.sqlobject.SQLTable;
import ca.sqlpower.swingui.ProgressWatcher;
import ca.sqlpower.swingui.SPSUtils;
@@ -382,6 +385,29 @@
cancelButton.setVisible(true);
}
};
+
+ /**
+ * Listens for changes to the result's start time so it can set the
status
+ * label correctly when the profiler has started profiling.
+ */
+ private final SPListener profileStartedListener = new
AbstractSPListener() {
+
+ @Override
+ public void propertyChanged(PropertyChangeEvent evt) {
+ if (evt.getPropertyName().equals("createStartTime") &&
result.getException() == null) {
+ if (result.getCreateStartTime() >= 0) {
+ statusLabel.setText(result.toString());
+ statusLabel.setForeground(null);
+ } else {
+ statusLabel.setText("Waiting to be profiled...");
+ statusLabel.setForeground(null);
+ }
+ } else if (evt.getPropertyName().equals("exception")) {
+ statusLabel.setText("Failed: " +
result.getException().getMessage());
+ statusLabel.setForeground(Color.RED);
+ }
+ }
+ };
/**
* Creates a profile row component that visualizes the given profile
@@ -465,6 +491,7 @@
pw.start();
result.addProfileResultListener(profileResultListener);
+ result.addSPListener(profileStartedListener);
add(progressBar, ComponentType.PROGRESS_BAR);
statusLabel.setVisible(false);
--
To unsubscribe, reply using "remove me" as the subject.
