sergiotarxz
/
owlcode.tech
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Adding the chance to monitor google data.

This commit is contained in:
Sergiotarxz 2023-05-07 18:10:55 +02:00
parent ff572e844b
commit 52067b5647
4 changed files with 162 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
lib/BurguillosInfo/Controller/Metrics.pm
View File

@ -37,8 +37,10 @@ sub stats {
} }
my $data = $tracking->get_global_data($self); my $data = $tracking->get_global_data($self);
my $data_per_url = $tracking->get_data_for_urls($self); my $data_per_url = $tracking->get_data_for_urls($self);
my $google_data = $tracking->get_google_data($self);
$self->_filter_data_per_url($data_per_url); $self->_filter_data_per_url($data_per_url);
$self->render(tracking_data => $data, tracking_by_url => $data_per_url); $self->_filter_data_per_url($google_data);
$self->render(tracking_data => $data, tracking_by_url => $data_per_url, google_data => $google_data);
} }
sub _filter_data_per_url($self, $data_per_url) { sub _filter_data_per_url($self, $data_per_url) {

42
lib/BurguillosInfo/DB/Migrations.pm
View File

@ -6,29 +6,31 @@ use strict;
use warnings; use warnings;
sub MIGRATIONS { sub MIGRATIONS {
return ( return (
'CREATE TABLE options ( 'CREATE TABLE options (
name TEXT PRIMARY KEY, name TEXT PRIMARY KEY,
value TEXT value TEXT
)', )',
'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"', 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"',
'CREATE TABLE paths ( 'CREATE TABLE paths (
path TEXT PRIMARY KEY, path TEXT PRIMARY KEY,
first_seen timestamp DEFAULT NOW() first_seen timestamp DEFAULT NOW()
)', )',
'CREATE TABLE requests ( 'CREATE TABLE requests (
uuid UUID DEFAULT uuid_generate_v4(), uuid UUID DEFAULT uuid_generate_v4(),
remote_address TEXT NOT NULL, remote_address TEXT NOT NULL,
user_agent TEXT NOT NULL, user_agent TEXT NOT NULL,
params JSON NOT NULL, params JSON NOT NULL,
date timestamp DEFAULT NOW(), date timestamp DEFAULT NOW(),
path TEXT, path TEXT,
FOREIGN KEY (path) REFERENCES paths(path) FOREIGN KEY (path) REFERENCES paths(path)
)', )',
'ALTER TABLE paths ADD column last_seen TIMESTAMP;', 'ALTER TABLE paths ADD column last_seen TIMESTAMP;',
'ALTER TABLE paths ALTER COLUMN last_seen SET DEFAULT NOW();', 'ALTER TABLE paths ALTER COLUMN last_seen SET DEFAULT NOW();',
'ALTER TABLE requests ADD PRIMARY KEY (uuid)', 'ALTER TABLE requests ADD PRIMARY KEY (uuid)',
'CREATE INDEX request_extra_index on requests (date, path);', 'CREATE INDEX request_extra_index on requests (date, path);',
); 'ALTER TABLE requests ADD column referer text;',
'CREATE INDEX request_referer_index on requests (referer);',
);
} }
1; 1;

178
lib/BurguillosInfo/Tracking.pm
View File

@ -19,7 +19,6 @@ SELECT COUNT(DISTINCT (remote_address, user_agent))
FROM requests FROM requests
EOF EOF
sub new { sub new {
my $class = shift; my $class = shift;
$app = shift; $app = shift;
@ -27,7 +26,7 @@ sub new {
return bless {}, $class; return bless {}, $class;
} }
sub _add_path($self, $url) { sub _add_path ( $self, $url ) {
my $dbh = BurguillosInfo::DB->connect($app); my $dbh = BurguillosInfo::DB->connect($app);
$dbh->do( <<'EOF', undef, $url ); $dbh->do( <<'EOF', undef, $url );
INSERT INTO paths (path) VALUES($1) INSERT INTO paths (path) VALUES($1)
@ -36,9 +35,9 @@ ON CONFLICT (path) DO
EOF EOF
} }
sub _update_null_last_seen_paths_if_any($self) { sub _update_null_last_seen_paths_if_any ($self) {
my $dbh = BurguillosInfo::DB->connect($app); my $dbh = BurguillosInfo::DB->connect($app);
$dbh->do( <<'EOF', undef); $dbh->do( <<'EOF', undef );
UPDATE paths UPDATE paths
SET last_seen = requests_for_path.last_date SET last_seen = requests_for_path.last_date
@ -51,12 +50,14 @@ WHERE paths.last_seen IS NULL AND requests_for_path.path = paths.path;
EOF EOF
} }
sub _register_request_query($self, $remote_address, $user_agent, $params_json, $path) { sub _register_request_query ( $self, $remote_address, $user_agent,
$params_json, $path, $referer )
{
my $dbh = BurguillosInfo::DB->connect($app); my $dbh = BurguillosInfo::DB->connect($app);
$dbh->do( $dbh->do(
<<'EOF', undef, $remote_address, $user_agent, $params_json, $path ); <<'EOF', undef, $remote_address, $user_agent, $params_json, $path, $referer );
INSERT INTO requests(remote_address, user_agent, params, path) INSERT INTO requests(remote_address, user_agent, params, path, referer)
VALUES (?, ?, ?, ?); VALUES (?, ?, ?, ?, ?);
EOF EOF
} }
@ -64,78 +65,121 @@ sub register_request {
my $self = shift; my $self = shift;
my $c = shift; my $c = shift;
my $path = $c->req->url->path; my $path = $c->req->url->path;
my $dbh = BurguillosInfo::DB->connect($app); my $dbh = BurguillosInfo::DB->connect($app);
$self->_add_path($path); $self->_add_path($path);
$self->_update_null_last_seen_paths_if_any(); $self->_update_null_last_seen_paths_if_any();
my $remote_address = $c->tx->remote_address; my $remote_address = $c->tx->remote_address;
my $user_agent = $c->req->headers->user_agent; my $user_agent = $c->req->headers->user_agent;
my $referer = $c->req->headers->referer // '';
my $params_json = encode_json( $c->req->params->to_hash ); my $params_json = encode_json( $c->req->params->to_hash );
$self->_register_request_query($remote_address, $user_agent, $params_json, $path); $self->_register_request_query( $remote_address, $user_agent, $params_json,
say "Registered $remote_address with user agent $user_agent visited $path with $params_json"; $path, $referer );
say
"Registered $remote_address with user agent $user_agent visited $path with $params_json";
} }
sub get_global_data { sub get_global_data {
my $self = shift; my $self = shift;
my $c = shift; my $c = shift;
my $app = $c->app; my $app = $c->app;
my $dbh = BurguillosInfo::DB->connect($app); my $dbh = BurguillosInfo::DB->connect($app);
my $data = $dbh->selectrow_hashref(<<"EOF", undef); my $data = $dbh->selectrow_hashref( <<"EOF", undef );
SELECT SELECT
( (
$SELECT_GLOBAL $SELECT_GLOBAL
where date > NOW() - interval '1 day' where date > NOW() - interval '1 day'
) as unique_ips_last_24_hours, ) as unique_ips_last_24_hours,
( (
$SELECT_GLOBAL $SELECT_GLOBAL
where date > NOW() - interval '1 week' where date > NOW() - interval '1 week'
) as unique_ips_last_week, ) as unique_ips_last_week,
( (
$SELECT_GLOBAL $SELECT_GLOBAL
where date > NOW() - interval '1 month' where date > NOW() - interval '1 month'
) as unique_ips_last_month; ) as unique_ips_last_month;
EOF EOF
return $data; return $data;
} }
sub get_data_for_urls { my $GOOGLE_SELECT = "$SELECT_GLOBAL
my $self = shift; where requests.path = paths.path
my $c = shift; and requests.referer IS NOT NULL
my $app = $c->app; and requests.referer ~* '^https?://(?:www\\.)?google\\.\\w'
my $dbh = BurguillosInfo::DB->connect($app); and date > NOW()";
my $data = $dbh->selectall_arrayref(<<"EOF", {Slice => {}});
SELECT paths.path, sub get_google_data {
( my $self = shift;
$SELECT_GLOBAL my $c = shift;
where requests.path = paths.path and date > NOW() - interval '1 hour' my $app = $c->app;
) as unique_ips_last_1_hour, my $dbh = BurguillosInfo::DB->connect($app);
( my $data = $dbh->selectall_arrayref(<<"EOF", { Slice => {} } );
$SELECT_GLOBAL SELECT paths.path,
where requests.path = paths.path and date > NOW() - interval '3 hour' (
) as unique_ips_last_3_hours, $GOOGLE_SELECT - interval '1 hour'
( ) as unique_ips_last_1_hour,
$SELECT_GLOBAL (
where requests.path = paths.path and date > NOW() - interval '6 hour' $GOOGLE_SELECT - interval '3 hour'
) as unique_ips_last_6_hours, ) as unique_ips_last_3_hours,
( (
$SELECT_GLOBAL $GOOGLE_SELECT - interval '6 hour'
where requests.path = paths.path and date > NOW() - interval '12 hour' ) as unique_ips_last_6_hours,
) as unique_ips_last_12_hours, (
( $GOOGLE_SELECT - interval '12 hour'
$SELECT_GLOBAL ) as unique_ips_last_12_hours,
where requests.path = paths.path and date > NOW() - interval '1 day' (
) as unique_ips_last_24_hours, $GOOGLE_SELECT - interval '1 day'
( ) as unique_ips_last_24_hours,
$SELECT_GLOBAL (
where requests.path = paths.path and date > NOW() - interval '1 week' $GOOGLE_SELECT - interval '1 week'
) as unique_ips_last_week, ) as unique_ips_last_week,
( (
$SELECT_GLOBAL $GOOGLE_SELECT - interval '1 month'
where requests.path = paths.path and date > NOW() - interval '1 month' ) as unique_ips_last_month
) as unique_ips_last_month
FROM paths FROM paths
WHERE paths.last_seen > NOW() - INTERVAL '1 month'; WHERE paths.last_seen > NOW() - INTERVAL '1 month';
EOF EOF
return $data; return $data;
}
sub get_data_for_urls {
my $self = shift;
my $c = shift;
my $app = $c->app;
my $dbh = BurguillosInfo::DB->connect($app);
my $data = $dbh->selectall_arrayref( <<"EOF", { Slice => {} } );
SELECT paths.path,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '1 hour'
) as unique_ips_last_1_hour,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '3 hour'
) as unique_ips_last_3_hours,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '6 hour'
) as unique_ips_last_6_hours,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '12 hour'
) as unique_ips_last_12_hours,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '1 day'
) as unique_ips_last_24_hours,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '1 week'
) as unique_ips_last_week,
(
$SELECT_GLOBAL
where requests.path = paths.path and date > NOW() - interval '1 month'
) as unique_ips_last_month
FROM paths
WHERE paths.last_seen > NOW() - INTERVAL '1 month';
EOF
return $data;
} }
1; 1;

26
templates/metrics/stats.html.ep
View File

@ -32,4 +32,30 @@
</tr> </tr>
%} %}
</table> </table>
<h3>Google data.</h3>
<table>
<tr>
<th>Path</th>
<th data-sort-method="number">Visitors 1h</th>
<th data-sort-method="number">Visitors 3h</th>
<th data-sort-method="number">Visitors 6h</th>
<th data-sort-method="number">Visitors 12h</th>
<th data-sort-method="number">Visitors 24h</th>
<th data-sort-method="number">Visitors week</th>
<th data-sort-method="number">Visitors month</th>
</tr>
% for my $row (@$google_data) {
<tr>
<td><%="${base_url}$row->{path}"%></td>
<td><%=$row->{unique_ips_last_1_hour}%></td>
<td><%=$row->{unique_ips_last_3_hours}%></td>
<td><%=$row->{unique_ips_last_6_hours}%></td>
<td><%=$row->{unique_ips_last_12_hours}%></td>
<td><%=$row->{unique_ips_last_24_hours}%></td>
<td><%=$row->{unique_ips_last_week}%></td>
<td><%=$row->{unique_ips_last_month}%></td>
</tr>
%}
</table>
</div> </div>