...
 
Commits (2)
......@@ -2,6 +2,7 @@
namespace Minds\Controllers\Cli;
use Elasticsearch\Common\Exceptions\ServerErrorResponseException;
use Minds\Cli;
use Minds\Core;
use Minds\Entities;
......@@ -18,15 +19,18 @@ class Analytics extends Cli\Controller implements Interfaces\CliControllerInterf
case 'sync_activeUsers':
$this->out('Indexes user activity by guid and counts per day');
$this->out('--from={timestamp} the day to start counting. Default is yesterday at midnight');
$$his->out('--to={timestamp} the day to stop counting. Default is yesterday at midnight');
$this->out('--to={timestamp} the day to stop counting. Default is yesterday at midnight');
$this->out('--rangeOffset={number of days} the number of days to look back into the past. Default is 7');
$this->out('--mode={silent | notify} silent mode does not send emails when running batches to re-index. Notify sends the notifications. Default is notify');
break;
case 'sync_graphs':
$this->out('sync graphs between es and cassandra');
break;
case 'counts':
$this->out('Prints the counts of a user');
$this->out('--from={timestamp in milliseconds} the day to start count. Default is yesterday');
$this->out('--guid={user guid} REQUIRED the user to aggregate');
// no break
// no break
default:
$this->out('Syntax usage: cli analytics <type>');
$this->displayCommandHelp();
......@@ -50,7 +54,7 @@ class Analytics extends Cli\Controller implements Interfaces\CliControllerInterf
$this->out('Collecting user activity');
$this->out("Running in {$mode} mode");
while ($from <= $to) {
$this->out('Syncing for '.gmdate('c', $from));
$this->out('Syncing for ' . gmdate('c', $from));
$manager = new Core\Analytics\UserStates\Manager();
$manager->setReferenceDate($from)
->setRangeOffset($rangeOffset)
......@@ -79,4 +83,51 @@ class Analytics extends Cli\Controller implements Interfaces\CliControllerInterf
var_dump($result);
}
public function sync_graphs()
{
error_reporting(E_ALL);
ini_set('display_errors', 1);
/** @var Core\Analytics\Graphs\Manager $manager */
$manager = Core\Di\Di::_()->get('Analytics\Graphs\Manager');
$aggregates = [
'avgpageviews',
// 'interactions',
'offchainboosts',
'onchainboosts',
'offchainplus',
'onchainplus',
'offchainwire',
'onchainwire',
'activeusers',
'posts',
'votes',
'comments',
'reminds',
//'subscribers',
'totalpageviews',
'usersegments',
'pageviews',
'withdraw',
'tokensales',
'rewards',
];
if ($this->getOpt('aggregate')) {
$aggregates = [ $this->getOpt('aggregate') ];
}
foreach ($aggregates as $aggregate) {
$this->out("Syncing {$aggregate}");
$manager->sync([
'aggregate' => $aggregate,
'all' => true,
]);
}
$this->out('Completed caching site metrics');
}
}
<?php
namespace Minds\Controllers\api\v2;
use Minds\Api\Factory;
use Minds\Core;
use Minds\Core\Di\Di;
use Minds\Interfaces;
class analytics implements Interfaces\Api, Interfaces\ApiIgnorePam
{
public function get($pages)
{
if (!isset($pages[0])) {
return Factory::response([
'status' => 'error',
'message' => 'report must be provided'
]);
}
$span = 12;
$unit = 'month';
switch ($_GET['timespan'] ?? null) {
case 'hourly':
$span = 24;
$unit = 'hour';
break;
case 'daily':
$span = 7;
$unit = 'day';
break;
case 'monthly':
$span = 12;
$unit = 'month';
break;
}
/** @var Core\Analytics\Metrics\Manager $manager */
$manager = Di::_()->get('Analytics\Graphs\Manager');
try {
$urn = "urn:graph:" . $manager::buildKey([
'aggregate' => $pages[0],
'key' => $_GET['key'] ?? null,
'span' => $span,
'unit' => $unit,
]);
$graph = $manager->get($urn);
if (!$graph) {
throw new \Exception("Graph not found");
}
$data = $graph->getData();
} catch (\Exception $e) {
error_log($e);
return Factory::response([
'status' => 'error',
'message' => $e->getMessage()
]);
}
return Factory::response([
'status' => 'success',
'data' => $data
]);
}
public function post($pages)
{
// TODO: Implement post() method.
}
public function put($pages)
{
// TODO: Implement put() method.
}
public function delete($pages)
{
// TODO: Implement delete() method.
}
}
......@@ -27,7 +27,7 @@ class pageview implements Interfaces\Api, Interfaces\ApiIgnorePam
]);
}
if(!isset($_COOKIE['mwa'])) {
if (!isset($_COOKIE['mwa'])) {
//@TODO make this more unique
$id = uniqid(true);
......
<?php
namespace Minds\Core\Analytics;
use Minds\Core\Analytics\Graphs;
use Minds\Core\Di\Provider;
class AnalyticsProvider extends Provider
{
public function register()
{
$this->di->bind('Analytics\Graphs\Manager', function ($di) {
return new Graphs\Manager();
}, ['useFactory' => true]);
$this->di->bind('Analytics\Graphs\Repository', function ($di) {
return new Graphs\Repository();
}, ['useFactory' => true]);
}
}
This diff is collapsed.
<?php
namespace Minds\Core\Analytics\Graphs\Aggregates;
interface AggregateInterface
{
public function fetch(array $options = []);
}
This diff is collapsed.
<?php
/**
* Amount of comments a user did in a given time period
*/
namespace Minds\Core\Analytics\Graphs\Aggregates;
use DateTime;
use Minds\Core\Data\cache\abstractCacher;
use Minds\Core\Data\ElasticSearch\Client;
use Minds\Core\Data\ElasticSearch\Prepared\Search;
use Minds\Core\Di\Di;
use Minds\Core\Analytics\Graphs\Manager;
class Comments implements AggregateInterface
{
/** @var Client */
protected $client;
/** @var abstractCacher */
protected $cacher;
/** @var string */
protected $index;
/** @var string */
protected $dateFormat;
public function __construct($client = null, $cacher = null, $config = null)
{
$this->client = $client ?: Di::_()->get('Database\ElasticSearch');
$this->cacher = $cacher ?: Di::_()->get('Cache\Redis');
$this->index = $config ? $config->get('elasticsearch')['index'] : Di::_()->get('Config')->get('elasticsearch')['metrics_index'] . '-*';
}
/**
* Fetch all
* @param array $opts
* @return array
*/
public function fetchAll($opts = [])
{
$result = [];
foreach ([ 'hour', 'day', 'month' ] as $unit) {
$k = Manager::buildKey([
'aggregate' => $opts['aggregate'] ?? 'comments',
'key' => null,
'unit' => $unit,
]);
$result[$k] = $this->fetch([
'unit' => $unit,
]);
}
return $result;
}
public function fetch(array $options = [])
{
$options = array_merge([
'span' => 12,
'unit' => 'month', // day / month
'userGuid' => null,
], $options);
$userGuid = $options['userGuid'];
$from = null;
switch ($options['unit']) {
case "hour":
$from = (new DateTime('midnight'))->modify("-{$options['span']} hours");
$to = (new DateTime('midnight'));
$interval = '1h';
$this->dateFormat = 'y-m-d H:i';
break;
case "day":
$from = (new DateTime('midnight'))->modify("-{$options['span']} days");
$to = (new DateTime('midnight'));
$interval = '1d';
$this->dateFormat = 'y-m-d';
break;
case "month":
$from = (new DateTime('midnight first day of next month'))->modify("-{$options['span']} months");
$to = new DateTime('midnight first day of next month');
$interval = '1M';
$this->dateFormat = 'y-m';
break;
default:
throw new \Exception("{$options['unit']} is not an accepted unit");
}
$result = null;
$query = [
'index' => $this->index,
'size' => 0,
"stored_fields" => [
"*"
],
"docvalue_fields" => [
(object) [
"field" => "@timestamp",
"format" => "date_time"
]
],
'body' => [
'query' => [
'bool' => [
'must' => [
[
'match_all' => (object) []
],
[
'range' => [
'@timestamp' => [
'gte' => $from->getTimestamp() * 1000,
'lte' => $to->getTimestamp() * 1000,
'format' => 'epoch_millis'
]
]
],
[
"match_phrase" => [
"action.keyword" => [
"query" => "comment"
]
]
]
]
]
],
'aggs' => [
'comments' => [
'date_histogram' => [
'field' => '@timestamp',
'interval' => $interval,
'min_doc_count' => 1,
],
"aggs" => [
"uniques" => [
"cardinality" => [
"field" => "user_guid.keyword"
]
]
]
]
]
]
];
if ($userGuid) {
$query['body']['query']['bool']['must'][] = [
'match' => [
'entity_owner_guid.keyword' => $userGuid
]
];
}
$prepared = new Search();
$prepared->query($query);
$result = $this->client->request($prepared);
$response = [
[
'name' => 'Comments',
'x' => [],
'y' => []
],
];
if (!$userGuid) {
$response[] = [
'name' => 'Number of Commenting Users',
'x' => [],
'y' => []
];
}
foreach ($result['aggregations']['comments']['buckets'] as $count) {
$date = date($this->dateFormat, $count['key'] / 1000);
$response[0]['x'][] = $date;
$response[0]['y'][] = $count['doc_count'];
if (!$userGuid) {
$response[1]['x'][] = $date;
$response[1]['y'][] = $count['uniques']['value'];
}
}
return $response;
}
public function hasTTL(array $opts = [])
{
return isset($opts['userGuid']);
}
public function buildCacheKey(array $opts = [])
{
return "comments:{$opts['unit']}:{$opts['userGuid']}";
}
}
<?php
namespace Minds\Core\Analytics\Graphs\Aggregates;
use DateTime;
use Minds\Core\Data\cache\abstractCacher;
use Minds\Core\Data\ElasticSearch\Client;
use Minds\Core\Data\ElasticSearch\Prepared\Count;
use Minds\Core\Data\ElasticSearch\Prepared\Search;
use Minds\Core\Di\Di;
use Minds\Core\Analytics\Graphs\Manager;
class Interactions implements AggregateInterface
{
/** @var Client */
protected $client;
/** @var abstractCacher */
protected $cacher;
/** @var string */
protected $index;
/** @var string */
protected $dateFormat;
public function __construct($client = null, $cacher = null, $config = null)
{
$this->client = $client ?: Di::_()->get('Database\ElasticSearch');
$this->cacher = $cacher ?: Di::_()->get('Cache\Redis');
$this->index = $config ? $config->get('elasticsearch')['index'] : Di::_()->get('Config')->get('elasticsearch')['metrics_index'] . '-*';
}
public function hasTTL(array $opts = [])
{
return true;
}
public function fetch(array $options = [])
{
$options = array_merge([
'span' => 1,
'unit' => 'day', // day / month
'key' => null,
'userGuid' => null,
], $options);
if (!isset($options['userGuid'])) {
throw new \Exception('userGuid must be set in the options parameter');
}
$userGuid = $options['userGuid'];
if (!isset($options['key'])) {
throw new \Exception('key must be set in the options parameter');
}
$key = $options['key'];
$from = null;
switch ($options['unit']) {
case "day":
$from = (new DateTime('midnight'))->modify("-{$options['span']} days");
$to = (new DateTime('midnight'));
$interval = '1d';
$this->dateFormat = 'y-m-d';
break;
case "month":
$from = (new DateTime('midnight first day of next month'))->modify("-{$options['span']} months");
$to = new DateTime('midnight first day of next month');
$interval = '1M';
$this->dateFormat = 'y-m';
break;
default:
throw new \Exception("{$options['unit']} is not an accepted unit");
}
$result = null;
$boost = strpos($key, 'boost') !== false;
if (strpos($key, 'totals') !== false) {
$result = $this->getTotals($from, $to, $boost, $userGuid);
} else {
$result = $this->getMetrics($from, $to, $boost, $userGuid, $interval);
}
return $result;
}
public function buildCacheKey(array $options = [])
{
if (!isset($options['userGuid'])) {
throw new \Exception('userGuid must be set in the options parameter');
}
if (!isset($options['key'])) {
throw new \Exception('key must be set in the options parameter');
}
return "interactions:{$options['key']}:{$options['unit']}:{$options['userGuid']}";
}
private function getTotals($from, $to, $boost, $userGuid)
{
return [
[
'values' => [
$this->getTotal('vote:up', $from, $to, $boost, $userGuid),
$this->getTotal('vote:down', $from, $to, $boost, $userGuid),
$this->getTotal('comment', $from, $to, $boost, $userGuid),
$this->getTotal('remind', $from, $to, $boost, $userGuid),
],
'labels' => [
'Vote Up', 'Vote Down', 'Comment', 'Remind'
]
]
];
}
private function getTotal($action, $from, $to, $boost, $userGuid)
{
$query = [
'index' => $this->index,
'type' => 'action',
'body' => [
'query' => [
'bool' => [
'filter' => [
'term' => [
'action' => $action
]
],
'must_not' => [
'term' => [
'user_guid' => $userGuid // don't count self interactions
]
],
'must' => [
[
'match_all' => (object) []
],
[
'range' => [
'@timestamp' => [
'gte' => $from->getTimestamp() * 1000,
'lte' => $to->getTimestamp() * 1000,
'format' => 'epoch_millis'
]
]
],
[
'match' => [
'entity_owner_guid' => $userGuid
]
]
]
]
]
]
];
if ($boost) {
$query['body']['query']['bool']['must'][] = [
'exists' => [
'field' => 'client_meta_campaign'
]
];
}
$prepared = new Count();
$prepared->query($query);
$result = $this->client->request($prepared);
return $result['count'] ?? 0;
}
private function getMetrics($from, $to, $boost, $userGuid, $interval)
{
$query = [
'index' => $this->index,
'size' => 0,
'type' => 'action',
'body' => [
'query' => [
'bool' => [
'filter' => [
[
'terms' => [
'action' => ['vote:up', 'vote:down', 'comment', 'remind']
]
]
],
'must_not' => [
'term' => [
'user_guid' => $userGuid // don't count self interactions
]
],
'must' => [
[
'match_all' => (object) []
],
[
'range' => [
'@timestamp' => [
'gte' => $from->getTimestamp() * 1000,
'lte' => $to->getTimestamp() * 1000,
'format' => 'epoch_millis'
]
]
],
[
'match' => [
'entity_owner_guid' => $userGuid
]
]
]
]
],
'aggs' => [
'interactions' => [
'date_histogram' => [
'field' => '@timestamp',
'interval' => $interval,
'min_doc_count' => 1,
]
]
]
]
];
$prepared = new Search();
$prepared->query($query);
$result = $this->client->request($prepared);
$response = [
[
'name' => 'Interactions',
'x' => [],
'y' => []
]
];
foreach ($result['aggregations']['interactions']['buckets'] as $count) {
$response[0]['x'][] = date($this->dateFormat, $count['key'] / 1000);
$response[0]['y'][] = $count['doc_count'];
}
if ($boost) {
$query['body']['query']['bool']['must'][] = [
'exists' => [
'field' => 'client_meta_campaign'
]
];
$prepared = new Search();
$prepared->query($query);
$boostResult = $this->client->request($prepared);
$response[] = [
[
'name' => 'Boost Interactions',
'x' => [],
'y' => []
]
];
foreach ($boostResult['aggregations']['interactions']['buckets'] as $count) {
$response[1]['x'][] = date($this->dateFormat, $count['key'] / 1000);
$response[1]['y'][] = $count['doc_count'];
}
}
return $response;
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
<?php
namespace Minds\Core\Analytics\Graphs\Aggregates;
use DateTime;
use Minds\Core\Data\cache\abstractCacher;
use Minds\Core\Data\ElasticSearch;
use Minds\Core\Data\ElasticSearch\Client;
use Minds\Core\Di\Di;
use Minds\Core\Analytics\Graphs\Manager;
class Pageviews implements AggregateInterface
{
/** @var Client */
protected $client;
/** @var abstractCacher */
protected $cacher;
/** @var string */
protected $index;
/** @var string */
protected $dateFormat;
public function __construct($client = null, $cacher = null, $config = null)
{
$this->client = $client ?: Di::_()->get('Database\ElasticSearch');
$this->cacher = $cacher ?: Di::_()->get('Cache\Redis');
$this->index = $config ? $config->get('elasticsearch')['index'] : Di::_()->get('Config')->get('elasticsearch')['metrics_index'] . '-*';
}
/**
* Fetch all
* @param array $opts
* @return array
*/
public function fetchAll($opts = [])
{
$result = [];
foreach ([
null,
'routes',
] as $key) {
foreach ([ 'day', 'month' ] as $unit) {
$k = Manager::buildKey([
'aggregate' => $opts['aggregate'] ?? 'pageviews',
'key' => $key,
'unit' => $unit,
]);
$result[$k] = $this->fetch([
'key' => $key,
'unit' => $unit,
]);
}
}
return $result;
}
public function fetch(array $options = [])
{
$options = array_merge([
'span' => 12,
'unit' => 'month', // day / month
'key' => null,
], $options);
$key = $options['key'];
$from = null;
switch ($options['unit']) {
case "day":
$from = (new DateTime('midnight'))->modify("-{$options['span']} days");
$to = (new DateTime('midnight'));
$interval = '1d';
$this->dateFormat = 'y-m-d';
break;
case "month":
$from = (new DateTime('midnight first day of next month'))->modify("-{$options['span']} months");
$to = new DateTime('midnight first day of next month');
$interval = '1M';
$this->dateFormat = 'y-m';
break;
default:
throw new \Exception("{$options['unit']} is not an accepted unit");
}
$response = null;
if ($key && $key == 'routes') {
$response = $this->getRoutes($from, $to, $interval);
} else {
$response = $this->getGraph($from, $to, $interval);
}
return $response;
}
public function hasTTL(array $opts = [])
{
return false;
}
public function buildCacheKey(array $opts = [])
{
return "pageviews:{$opts['key']}:{$opts['unit']}";
}
private function getRoutes($from, $to, $interval)
{
$must = [
[
"match_all" => (object) []
],
[
"range" => [
"@timestamp" => [
"gte" => $from->getTimestamp() * 1000,
"lte" => $to->getTimestamp() * 1000,
"format" => "epoch_millis"
]
]
],
[
'match_phrase' => [
'action.keyword' => [
'query' => 'pageview'
]
]
]
];
$query = [
'index' => $this->index,
'size' => 0,
"stored_fields" => [
"*"
],
"docvalue_fields" => [
(object) [
"field" => "@timestamp",
"format" => "date_time"
]
],
'body' => [
'query' => [
'bool' => [
'must' => $must
]
],
"aggs" => [
"routes" => [
"terms" => [
"field" => "route_uri.keyword",
'size' => 9,
'order' => [
'_count' => 'desc'
]
]
],
]
]
];
$prepared = new ElasticSearch\Prepared\Search();
$prepared->query($query);
$result = $this->client->request($prepared);
$response = [
[
'name' => 'Pageviews by Route',
'values' => [],
'labels' => []
]
];
foreach ($result['aggregations']['routes']['buckets'] as $count) {
$response[0]['labels'][] = $count['key'];
$response[0]['values'][] = $count['doc_count'];
}
return $response;
}
private function getGraph($from, $to, $interval)
{
$must = [
[
"match_all" => (object) []
],
[
"range" => [
"@timestamp" => [
"gte" => $from->getTimestamp() * 1000,
"lte" => $to->getTimestamp() * 1000,
"format" => "epoch_millis"
]
]
],
[
'match_phrase' => [
'action.keyword' => [
'query' => 'pageview'
]
]
]
];
$query = [
'index' => $this->index,
'size' => 0,
"stored_fields" => [
"*"
],
"docvalue_fields" => [
(object) [
"field" => "@timestamp",
"format" => "date_time"
]
],
'body' => [
'query' => [
'bool' => [
'must' => $must
]
],
"aggs" => [
"histogram" => [
"date_histogram" => [
"field" => "@timestamp",
'interval' => $interval,
'min_doc_count' => 1
]
],
]
]
];
$prepared = new ElasticSearch\Prepared\Search();
$prepared->query($query);
$result = $this->client->request($prepared);
$response = [
[
'name' => 'Number of Pageviews',
'x' => [],
'y' => []
]
];
foreach ($result['aggregations']['histogram']['buckets'] as $count) {
$response[0]['x'][] = date($this->dateFormat, $count['key'] / 1000);
$response[0]['y'][] = $count['doc_count'];
}
return $response;
}
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
<?php
/**
* Impression Metric
*/
* Impression Metric
*/
namespace Minds\Core\Analytics\Metrics;
use Minds\Helpers;
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.