[Piwik-svn] r504 - in trunk: modules modules/ArchiveProcessing plugins/Actions plugins/Referers
svnmaster at piwik.org
svnmaster at piwik.org
Sun Jun 1 22:19:32 CEST 2008
Author: matt
Date: 2008-06-01 22:19:28 +0200 (Sun, 01 Jun 2008)
New Revision: 504
Modified:
trunk/modules/ArchiveProcessing/Day.php
trunk/modules/ArchiveProcessing/Period.php
trunk/modules/DataTable.php
trunk/plugins/Actions/Actions.php
trunk/plugins/Referers/Referers.php
Log:
- making things faster for the 2 bottlenecks: actions & referers
Modified: trunk/modules/ArchiveProcessing/Day.php
===================================================================
--- trunk/modules/ArchiveProcessing/Day.php 2008-06-01 19:16:56 UTC (rev 503)
+++ trunk/modules/ArchiveProcessing/Day.php 2008-06-01 20:19:28 UTC (rev 504)
@@ -275,7 +275,7 @@
* @param array of Piwik_DataTable $subArrayLevel1ByKey
* @return array Array with N elements: the strings of the datatable serialized
*/
- public function getDataTablesSerialized( $arrayLevel0, $subArrayLevel1ByKey)
+ public function getDataTablesSerialized( $arrayLevel0, $subArrayLevel1ByKey, $maximumRowsInDataTableLevelZero = null, $maximumRowsInSubDataTable = null)
{
$tablesByLabel = array();
@@ -288,7 +288,7 @@
$parentTableLevel0 = new Piwik_DataTable;
$parentTableLevel0->loadFromArrayLabelIsKey($subArrayLevel1ByKey, $tablesByLabel);
- $toReturn = $parentTableLevel0->getSerialized();
+ $toReturn = $parentTableLevel0->getSerialized($maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
return $toReturn;
}
Modified: trunk/modules/ArchiveProcessing/Period.php
===================================================================
--- trunk/modules/ArchiveProcessing/Period.php 2008-06-01 19:16:56 UTC (rev 503)
+++ trunk/modules/ArchiveProcessing/Period.php 2008-06-01 20:19:28 UTC (rev 504)
@@ -140,7 +140,7 @@
* nameTable2 => number of rows,
* )
*/
- public function archiveDataTable( $aRecordName )
+ public function archiveDataTable( $aRecordName, $maximumRowsInDataTableLevelZero = null, $maximumRowsInSubDataTable = null )
{
if(!is_array($aRecordName))
{
@@ -155,7 +155,7 @@
$nameToCount[$recordName]['level0'] = $table->getRowsCount();
$nameToCount[$recordName]['recursive'] = $table->getRowsCountRecursive();
- $record = new Piwik_ArchiveProcessing_Record_BlobArray($recordName, $table->getSerialized());
+ $record = new Piwik_ArchiveProcessing_Record_BlobArray($recordName, $table->getSerialized( $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable ));
}
return $nameToCount;
}
Modified: trunk/modules/DataTable.php
===================================================================
--- trunk/modules/DataTable.php 2008-06-01 19:16:56 UTC (rev 503)
+++ trunk/modules/DataTable.php 2008-06-01 20:19:28 UTC (rev 504)
@@ -715,6 +715,9 @@
* won't work.
*
* @throws Exception if an infinite recursion is found (a table row's has a subtable that is one of its parent table)
+ * @param int If not null, defines the number of rows maximum of the serialized dataTable
+ * If $addSummaryRowAfterNRows is less than the size of the table, a SummaryRow will be added at the end of the table, that
+ * is the sum of the values of all the rows after the Nth row. All the rows after the Nth row will be deleted.
*
* @return array Serialized arrays
* array( // Datatable level0
@@ -729,8 +732,9 @@
* //first Datatable level3 (child of second Datatable level1 for example)
* 3 => 'eghuighahgaueytae78yaet7yaetaeGRQWUBGUIQGH&QE',
* );
+ *
*/
- public function getSerialized()
+ public function getSerialized( $maximumRowsInDataTable = null, $maximumRowsInSubDataTable = null )
{
static $depth = 0;
@@ -749,7 +753,7 @@
{
$subTable = Piwik_DataTable_Manager::getInstance()->getTable($idSubTable);
$depth++;
- $aSerializedDataTable = $aSerializedDataTable + $subTable->getSerialized();
+ $aSerializedDataTable = $aSerializedDataTable + $subTable->getSerialized( $maximumRowsInSubDataTable, $maximumRowsInSubDataTable );
$depth--;
}
}
@@ -762,8 +766,10 @@
$forcedId = 0;
}
- //TODO limit temporary here, will be set on a per plugin basis
- $filter = new Piwik_DataTable_Filter_AddSummaryRow($this, 500);
+ if( !is_null($maximumRowsInDataTable) )
+ {
+ $filter = new Piwik_DataTable_Filter_AddSummaryRow($this, $maximumRowsInDataTable - 1);
+ }
// we then serialize the rows and store them in the serialized dataTable
$aSerializedDataTable[$forcedId] = serialize($this->rows + array( self::ID_SUMMARY_ROW => $this->summaryRow));
Modified: trunk/plugins/Actions/Actions.php
===================================================================
--- trunk/plugins/Actions/Actions.php 2008-06-01 19:16:56 UTC (rev 503)
+++ trunk/plugins/Actions/Actions.php 2008-06-01 20:19:28 UTC (rev 504)
@@ -79,8 +79,10 @@
'Actions_downloads',
'Actions_outlink',
);
-
- $archiveProcessing->archiveDataTable($dataTableToSum);
+
+ $maximumRowsInDataTableLevelZero = 200;
+ $maximumRowsInSubDataTable = 50;
+ $archiveProcessing->archiveDataTable($dataTableToSum, $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
}
/**
@@ -189,18 +191,22 @@
$query = $archiveProcessing->db->query($query, array( $archiveProcessing->strDateStart, $archiveProcessing->idsite ));
$modified = $this->updateActionsTableWithRowQuery($query);
-
+
+ $maximumRowsInDataTableLevelZero = 200;
+ $maximumRowsInSubDataTable = 50;
+
$dataTable = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_ACTION]);
- $s = $dataTable->getSerialized();
+ $s = $dataTable->getSerialized( $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable );
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Actions_actions', $s);
$dataTable = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_DOWNLOAD]);
- $s = $dataTable->getSerialized();
+ $s = $dataTable->getSerialized( $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable );
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Actions_downloads', $s);
$dataTable = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_OUTLINK]);
- $s = $dataTable->getSerialized();
+ $s = $dataTable->getSerialized( $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable );
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Actions_outlink', $s);
+
unset($this->actionsTablesByType);
}
Modified: trunk/plugins/Referers/Referers.php
===================================================================
--- trunk/plugins/Referers/Referers.php 2008-06-01 19:16:56 UTC (rev 503)
+++ trunk/plugins/Referers/Referers.php 2008-06-01 20:19:28 UTC (rev 504)
@@ -73,9 +73,11 @@
'Referers_urlByWebsite',
'Referers_urlByPartner',
);
+
+ $maximumRowsInDataTableLevelZero = 500;
+ $maximumRowsInSubDataTable = 50;
+ $nameToCount = $archiveProcessing->archiveDataTable($dataTableToSum, $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
- $nameToCount = $archiveProcessing->archiveDataTable($dataTableToSum);
-
$mappingFromArchiveName = array(
'Referers_distinctSearchEngines' =>
array( 'typeCountToUse' => 'level0',
@@ -130,9 +132,6 @@
}
}
- /**
- *
- */
public function archiveDay( $notification )
{
$archiveProcessing = $notification->getNotificationObject();
@@ -236,37 +235,7 @@
if(!isset($interestByType[$row['referer_type']] )) $interestByType[$row['referer_type']] = $archiveProcessing->getNewInterestRow();
$archiveProcessing->updateInterestStats($row, $interestByType[$row['referer_type']]);
}
-// echo "after loop = ". $timer;
-// Piwik::log("By search engine:");
-// Piwik::log($interestBySearchEngine);
-// Piwik::log("By keyword:");
-// Piwik::log($interestByKeyword);
-// Piwik::log("Kwd by search engine:");
-// Piwik::log($keywordBySearchEngine);
-// Piwik::log("Search engine by keyword:");
-// Piwik::log($searchEngineByKeyword);
-//
-
-// Piwik::log("By campaign:");
-// Piwik::log($interestByCampaign);
-// Piwik::log("Kwd by campaign:");
-// Piwik::log($keywordByCampaign);
-
-
-// Piwik::log("By referer type:");
-// Piwik::log($interestByType);
-
-// Piwik::log("By website:");
-// Piwik::log($interestByWebsite[Piwik_Common::REFERER_TYPE_WEBSITE]);
-// Piwik::log("Urls by website:");
-// Piwik::log($urlByWebsite[Piwik_Common::REFERER_TYPE_WEBSITE]);
-// Piwik::log("By partner website:");
-// Piwik::log($interestByWebsite[Piwik_Common::REFERER_TYPE_PARTNER]);
-// Piwik::log("Urls by partner website:");
-// Piwik::log($urlByWebsite[Piwik_Common::REFERER_TYPE_PARTNER]);
-
-
$numberOfDistinctSearchEngines = count($keywordBySearchEngine);
$numberOfDistinctKeywords = count($searchEngineByKeyword);
@@ -290,30 +259,26 @@
$record = new Piwik_ArchiveProcessing_Record_Numeric($name, $value);
}
-// Piwik::printMemoryUsage("Middle of ".get_class($this)." ");
-
$data = $archiveProcessing->getDataTableSerialized($interestByType);
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Referers_type', $data);
+
+ $maximumRowsInDataTableLevelZero = 500;
+ $maximumRowsInSubDataTable = 50;
- $data = $archiveProcessing->getDataTablesSerialized($keywordBySearchEngine, $interestBySearchEngine);
+ $data = $archiveProcessing->getDataTablesSerialized($keywordBySearchEngine, $interestBySearchEngine, $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Referers_keywordBySearchEngine', $data);
-// var_export($data);
-
- $data = $archiveProcessing->getDataTablesSerialized($searchEngineByKeyword, $interestByKeyword);
+ $data = $archiveProcessing->getDataTablesSerialized($searchEngineByKeyword, $interestByKeyword, $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Referers_searchEngineByKeyword', $data);
- $data = $archiveProcessing->getDataTablesSerialized($keywordByCampaign, $interestByCampaign);
+ $data = $archiveProcessing->getDataTablesSerialized($keywordByCampaign, $interestByCampaign, $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Referers_keywordByCampaign', $data);
- $data = $archiveProcessing->getDataTablesSerialized($urlByWebsite[Piwik_Common::REFERER_TYPE_WEBSITE], $interestByWebsite[Piwik_Common::REFERER_TYPE_WEBSITE]);
+ $data = $archiveProcessing->getDataTablesSerialized($urlByWebsite[Piwik_Common::REFERER_TYPE_WEBSITE], $interestByWebsite[Piwik_Common::REFERER_TYPE_WEBSITE], $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Referers_urlByWebsite', $data);
- $data = $archiveProcessing->getDataTablesSerialized($urlByWebsite[Piwik_Common::REFERER_TYPE_PARTNER], $interestByWebsite[Piwik_Common::REFERER_TYPE_PARTNER]);
+ $data = $archiveProcessing->getDataTablesSerialized($urlByWebsite[Piwik_Common::REFERER_TYPE_PARTNER], $interestByWebsite[Piwik_Common::REFERER_TYPE_PARTNER], $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable);
$record = new Piwik_ArchiveProcessing_Record_BlobArray('Referers_urlByPartner', $data);
-
-// Piwik::printMemoryUsage("End of ".get_class($this)." ");
-// echo "after serialization = ". $timer;
}
}
More information about the Piwik-svn
mailing list