Skip to content
Snippets Groups Projects
Commit eded5b65 authored by Allen Gilliland's avatar Allen Gilliland
Browse files

Merge branch 'master' into ag-csv-support

Conflicts:
	src/metabase/util.clj
parents 5235a167 357e20b9
No related branches found
No related tags found
No related merge requests found
Showing
with 65 additions and 1324 deletions
......@@ -17,7 +17,9 @@
<div class="border-bottom bg-white py1">
<div class="wrapper">
<label class="Select" ng-if="user.is_multi_org">
<select ng-change="changeCurrOrg(currentOrgSlug)" ng-model="currentOrgSlug" ng-options="perm.organization.slug as perm.organization.name for perm in user.org_perms"></select>
<select ng-change="changeCurrOrg(currentOrgSlug)" ng-model="currentOrgSlug"
ng-options="perm.organization.slug as perm.organization.name for perm in user.org_perms">
</select>
</label>
<a cv-org-href="/admin/">
<h3 class="AdminPageTitle my2 inline-block">
......@@ -45,9 +47,6 @@
<li>
<a class="NavItem" cv-org-href="/admin/datasets">Datasets</a>
</li>
<li>
<a class="NavItem" cv-org-href="/admin/datasources">DataSources</a>
</li>
<li>
<a class="NavItem" cv-org-href="/admin/emailreport/">Email Reports</a>
</li>
......@@ -62,17 +61,6 @@
</li>
</ul>
</li>
<li>
<a class="NavItem" cv-org-href="/admin/etl/jobexec/">ETL</a>
<ul class="ContextNav">
<li>
<a class="NavItem" cv-org-href="/admin/etl/jobexec/">Status</a>
</li>
<li>
<a class="NavItem" cv-org-href="/admin/etl/job/">Jobs</a>
</li>
</ul>
</li>
<li>
<a class="NavItem" cv-org-href="/admin/search">Search</a>
</li>
......@@ -161,12 +149,6 @@
<script src="/app/admin/query/query.module.js"></script>
<script src="/app/admin/query/query.controllers.js"></script>
<script src="/app/admin/query/query.services.js"></script>
<script src="/app/admin/etl/etl.module.js"></script>
<script src="/app/admin/etl/etl.controllers.js"></script>
<script src="/app/admin/etl/etl.services.js"></script>
<script src="/app/admin/datasources/datasources.module.js"></script>
<script src="/app/admin/datasources/datasources.controllers.js"></script>
<script src="/app/admin/datasources/datasources.services.js"></script>
<script src="/app/admin/annotation/annotation.module.js"></script>
<script src="/app/admin/annotation/annotation.controllers.js"></script>
<script src="/app/admin/search/search.module.js"></script>
......@@ -182,9 +164,6 @@
<script src="/app/dashboard/dashboard.controllers.js"></script>
<script src="/app/dashboard/dashboard.services.js"></script>
<script src="/app/dashboard/dashboard.directives.js"></script>
<script src="/app/etl/etl.module.js"></script>
<script src="/app/etl/etl.controllers.js"></script>
<script src="/app/etl/etl.services.js"></script>
<script src="/app/explore/explore.module.js"></script>
<script src="/app/explore/explore.controllers.js"></script>
<script src="/app/explore/explore.directives.js"></script>
......
......@@ -18,9 +18,7 @@ var CorvusAdmin = angular.module('corvusadmin', [
'corvusadmin.index.controllers',
'corvusadmin.databases',
'corvusadmin.datasets',
'corvusadmin.datasources',
'corvusadmin.emailreport',
'corvusadmin.etl',
'corvusadmin.people',
'corvusadmin.query',
'corvusadmin.annotation',
......
'use strict';
/*global _*/
// Card Controllers
var DataSourceControllers = angular.module('corvusadmin.datasources.controllers', []);
DataSourceControllers.controller('DataSourceList', ['$scope', '$location', 'DataSource', function($scope, $location, DataSource) {
$scope.delete = function (datasourceId) {
if ($scope.datasources) {
DataSource.delete({
'dataSourceId': datasourceId
}, function (result) {
$scope.datasources = _.filter($scope.datasources, function(datasource){
return datasource.id != datasourceId;
});
}, function (error) {
console.log('error deleting datasource', error);
});
}
};
$scope.$watch('currentOrg', function (org) {
if (!org) return;
DataSource.list({
'orgId': org.id
}, function (datasources) {
$scope.datasources = datasources;
}, function (error) {
console.log('error getting datasources list', error);
});
});
}]);
DataSourceControllers.controller('DataSourceDetail', ['$scope', '$routeParams', '$location', 'DataSource', 'SourceTypeHelpers', function($scope, $routeParams, $location, DataSource, SourceTypeHelpers) {
// $scope.datasource
// $scope.ingestions
// $scope.creation_information
$scope.save = function (datasource) {
// make sure the parameters are nice and tidy
var all_parameters = $scope.creation_information.available_datasources[datasource.source_type];
for(var parameter_id in all_parameters) {
var parameter = all_parameters[parameter_id];
if (parameter.parent && !datasource.parameters[parameter.parent]) {
delete datasource.parameters[parameter.name];
}
}
if ($scope.datasource.id) {
// if there is already an ID associated with the datasource then we are updating
DataSource.update(datasource, function (updated_datasource) {
$scope.datasource = updated_datasource;
}, function (error) {
console.log('error updating datasource', error);
});
} else {
// otherwise we are creating a new datasource
datasource.org = $scope.currentOrg.id;
DataSource.create(datasource, function (new_datasource) {
$location.path('/' + $scope.currentOrg.slug + '/admin/datasources/' + new_datasource.id);
}, function (error) {
console.log('error creating datasource', error);
});
}
};
$scope.ingest = function () {
// lets do it
DataSource.ingest({
'dataSourceId': $routeParams.dataSourceId
}, function (result) {
// put the new ingestion at the top of the list
$scope.ingestions.unshift(result);
}, function (error) {
console.log('error starting datasource ingestion', error);
});
};
$scope.setPage = function (page_number) {
$scope.page = page_number;
DataSource.ingestions({
'dataSourceId': $routeParams.dataSourceId,
'pageNumber': $scope.page
}, function (ingestions) {
$scope.ingestions = ingestions;
}, function (error) {
console.log('error getting datasource ingestions', error);
});
};
$scope.reingest = function () {
// lets do it
DataSource.reingest({
'dataSourceId': $routeParams.dataSourceId
}, function (result) {
// put the new ingestion at the top of the list
$scope.ingestions.unshift(result);
}, function (error) {
console.log('error starting datasource ingestion', error);
});
};
$scope.clearTypeParameters = function() {
// clear type parameters. Might be slicker to keep valid ones as you transition from type to type
$scope.datasource.parameters = {};
};
if ($routeParams.dataSourceId) {
$scope.page = 1;
// load existing datasource for editing
DataSource.get({
'dataSourceId': $routeParams.dataSourceId
}, function (datasource) {
$scope.datasource = datasource;
$scope.isTimestampedDataSource = SourceTypeHelpers.checkTimestampedDataSource(datasource.source_type);
// we also need to get the ingestions related to this datasource
DataSource.ingestions({
'dataSourceId': $routeParams.dataSourceId,
'pageNumber': $scope.page
}, function (ingestions) {
$scope.ingestions = ingestions;
}, function (error) {
console.log('error getting datasource ingestions', error);
});
}, function (error) {
console.log('error loading datasource', error);
if (error.status == 404) {
$location.path('/');
}
});
} else {
// prepare an empty datasource for creation
$scope.datasource = {};
}
// we will also need the data to populate our form elements
DataSource.creation_information(function (creation_info) {
$scope.creation_information = creation_info;
}, function (error) {
console.log('error getting datasource creation info', error);
});
}]);
DataSourceControllers.controller('DataSourceIngestionView', ['$scope', '$routeParams', '$location', 'DataSourceIngestion', 'DataSource', 'SourceTypeHelpers', function($scope, $routeParams, $location, DataSourceIngestion, DataSource, SourceTypeHelpers) {
// $scope.datasource_ingestion
// $scope.logs
if ($routeParams.dataSourceIngestionId) {
DataSourceIngestion.get({
'dataSourceIngestionId': $routeParams.dataSourceIngestionId
}, function (ingestion) {
$scope.datasource_ingestion = ingestion;
// get the datasource.source_type so we know to render the timestamp or not
DataSource.get({
'dataSourceId': ingestion.source
}, function (datasource) {
$scope.isTimestampedDataSource = SourceTypeHelpers.checkTimestampedDataSource(datasource.source_type);
}, function (error) {
console.log('error gettting datasource', error);
});
// now get the log details for this ingestion
DataSourceIngestion.log({
'dataSourceIngestionId': $routeParams.dataSourceIngestionId
}, function (captured_log) {
$scope.logs = captured_log;
}, function (error) {
console.log('error gettting datasource ingestion log', error);
});
}, function (error) {
console.log('error getting datasource ingestion', error);
if (error.status == 404) {
$location.path('/');
}
});
}
}]);
'use strict';
var Query = angular.module('corvusadmin.datasources', [
'corvusadmin.datasources.services',
'corvusadmin.datasources.controllers'
]);
Query.config(['$routeProvider', function ($routeProvider) {
$routeProvider.when('/:orgSlug/admin/datasources/', {templateUrl: '/app/admin/datasources/partials/datasource_list.html', controller: 'DataSourceList'});
$routeProvider.when('/:orgSlug/admin/datasources/create', {templateUrl: '/app/admin/datasources/partials/datasource_create.html', controller: 'DataSourceDetail'});
$routeProvider.when('/:orgSlug/admin/datasources/:dataSourceId', {templateUrl: '/app/admin/datasources/partials/datasource_view.html', controller: 'DataSourceDetail'});
$routeProvider.when('/:orgSlug/admin/datasources/:dataSourceId/modify', {templateUrl: '/app/admin/datasources/partials/datasource_modify.html', controller: 'DataSourceDetail'});
$routeProvider.when('/:orgSlug/admin/datasource_ingestion/:dataSourceIngestionId', {templateUrl: '/app/admin/datasources/partials/datasource_ingestion_view.html', controller: 'DataSourceIngestionView'});
$routeProvider.otherwise({redirectTo: '/:orgSlug/admin/'});
}]);
'use strict';
// Query Services
var DataSourceServices = angular.module('corvusadmin.datasources.services', ['ngResource', 'ngCookies']);
DataSourceServices.factory('DataSource', ['$resource', '$cookies', function($resource, $cookies) {
return $resource('/api/datasource/source/:dataSourceId', {}, {
list: {
url:'/api/datasource/source?org=:orgId',
method:'GET',
isArray:true
},
create: {
url:'/api/datasource/source',
method:'POST',
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
creation_information:{
url:'/api/datasource/source/creation_information/',
method:'GET',
},
get: {
method:'GET',
params:{dataSourceId:'@dataSourceId'}
},
update: {
method:'PUT',
params:{dataSourceId:'@id'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
delete: {
method:'DELETE',
params:{dataSourceId:'@dataSourceId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
ingest: {
method:'POST',
url:'/api/datasource/source/:dataSourceId/ingest',
params:{dataSourceId:'@dataSourceId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
reingest: {
method:'POST',
url:'/api/datasource/source/:dataSourceId/reingest',
params:{dataSourceId:'@dataSourceId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
ingestions: {
url:'/api/datasource/source/:dataSourceId/ingestions?page=:pageNumber',
method:'GET',
params:{dataSourceId:'@dataSourceId'},
isArray:true
}
});
}]);
DataSourceServices.factory('DataSourceIngestion', ['$resource', '$cookies', function($resource, $cookies) {
return $resource('/api/datasource/ingestion/:dataSourceIngestionId', {}, {
list: {
url:'/api/datasource/ingestion?org=:orgId',
method:'GET',
isArray:true
},
get: {
method:'GET',
params:{dataSourceIngestionId:'@dataSourceIngestionId'}
},
log: {
url:'/api/datasource/ingestion/:dataSourceIngestionId/log',
method:'GET',
params:{dataSourceId:'@dataSourceId'},
}
});
}]);
DataSourceServices.service('SourceTypeHelpers', [function(){
// Timestamped Datasources -- thanks @ded
this.checkTimestampedDataSource = function(source_type) {
return !!~['db_snap', 'log', 'xls', 'csv'].indexOf(source_type);
};
}]);
<div>
<h1>
Add Data Source
</h1>
<form id="datasource_form" novalidate>
<p>
<label for="id_name">Name:</label>
<input id="id_name" size="100" type="text" ng-model="datasource.name"/>
</p>
<p>
<label for="id_source_type">Type:</label>
<select id="id_source_type" ng-model="datasource.source_type" ng-options="type[0] as type[1] for type in creation_information.types" ng-disabled="readonly" ng-change="clearTypeParameters()">
<option value="">---------</option>
</select>
</p>
<p>
<label for="id_refresh_interval">Refresh Interval:</label>
<select id="id_refresh_interval" ng-model="datasource.refresh_interval" ng-options="interval[0] as interval[1] for interval in creation_information.refresh_intervals" ng-disabled="readonly">
<option value="">---------</option>
</select>
</p>
<p>
<label for="id_policy">Data Policy:</label>
<select id="id_policy" ng-model="datasource.data_policy" ng-options="policy[0] as policy[1] for policy in creation_information.data_policies" ng-disabled="readonly">
<option value="">---------</option>
</select>
</p>
<p>
<label for="id_datamart">Database:</label>
<select id="id_datamart" ng-model="datasource.database" ng-options="dm.id as dm.name for dm in creation_information.databases" ng-disabled="readonly">
<option value="">---------</option>
</select>
</p>
<span ng-repeat="parameter in creation_information.available_datasources[datasource.source_type]">
<p ng-if="!parameter.parent || datasource.parameters[parameter.parent]">
<label for="id_database">{{parameter.name}}</label>
<span ng-if="parameter.values.length > 0">
<label class="Select">
<select ng-model="datasource.parameters[parameter.name]" ng-options="f.key as f.name for f in field_for(table_filter, $index).values" ng-disabled="{{readonly}}">
</select>
</label>
</span>
<span ng-if="parameter.values.length == 0">
<input class="input" size="30" type="{{parameter.type}}" ng-model="datasource.parameters[parameter.name]" ng-readonly="{{readonly}}"/ >
</span>
</p>
</span>
<table width="100%">
<tr>
<td>
<a class="Button mx1" href="#" ng-click="save(datasource)">Add</a>
</td>
<td style="text-align: right">
</td>
</tr>
</table>
</form>
</div>
<div class=" clearfix">
<h2>Data Source Ingestion: {{datasource_ingestion.id}}</h2>
</div>
<div class="page-header row">
<div class="TableWrapper py4">
<table class="Table">
<thead>
<tr>
</tr>
</thead>
<tbody>
<tr>
<td>ID</td>
<td>{{datasource_ingestion.id}}</td>
</tr>
<tr>
<td>Source</td>
<td><a cv-org-href="/admin/datasources/{{datasource_ingestion.source}}">{{datasource_ingestion.source}}</a></td>
</tr>
<tr>
<td># Tables</td>
<td>{{datasource_ingestion.num_tables}}</td>
</tr>
<tr>
<td># Rows</td>
<td>{{datasource_ingestion.num_rows}}</td>
</tr>
<tr>
<td>Status</td>
<td>{{datasource_ingestion.status}}</td>
</tr>
<tr>
<td>Database</td>
<td>{{datasource_ingestion.database_name}}</td>
</tr>
<tr ng-if="isTimestampedDataSource">
<td>File Timestamp</td>
<td>{{datasource_ingestion.write_time}}</td>
</tr>
<tr>
<td>downloading_start</td>
<td>{{datasource_ingestion.downloading_start}}</td>
</tr>
<tr>
<td>downloading_end</td>
<td>{{datasource_ingestion.downloading_end}}</td>
</tr>
<tr>
<td>transformation_start</td>
<td>{{datasource_ingestion.transformation_start}}</td>
</tr>
<tr>
<td>transformation_end</td>
<td>{{datasource_ingestion.transformation_end}}</td>
</tr>
<tr>
<td>insertion_start</td>
<td>{{datasource_ingestion.insertion_start}}</td>
</tr>
<tr>
<td>insertion_end</td>
<td>{{datasource_ingestion.insertion_end}}</td>
</tr>
<tr>
<td>sync_start</td>
<td>{{datasource_ingestion.sync_start}}</td>
</tr>
<tr>
<td>sync_end</td>
<td>{{datasource_ingestion.sync_end}}</td>
</tr>
</tbody>
</table>
</div>
<div>
<h2>Captured Output</h2>
<pre>{{logs.captured_output}}</pre>
</div>
<div class=" clearfix">
<div class="float-right">
<div class="mx2 inline-block">
<a class="Button float-right" cv-org-href="/admin/datasources/create">Add Data Source</a>
</div>
</div>
<h2>Data Sources</h2>
</div>
<div class="page-content row">
<div class="TableWrapper">
<table class="Table">
<thead>
<th>Source</th>
<th>Type</th>
<th>Database</th>
<th>Refresh Interval</th>
<th>Data Policy</th>
<th>Created</th>
<th>Delete</th>
</thead>
<tr ng-if="!datasources">
<td colspan="7" style="text-align:center;"><h3>Loading ...</h3></td>
</tr>
<tr ng-repeat="datasource in datasources">
<td><a cv-org-href="/admin/datasources/{{datasource.id}}">{{datasource.name}}</a></td>
<td>{{datasource.source_type}}</td>
<td>{{datasource.database.name}}</td>
<td>{{datasource.refresh_interval}}</td>
<td>{{datasource.data_policy}}</td>
<td>{{datasource.created_at}}</td>
<td><a class="Button" ng-click="delete(datasource.id)" delete-confirm>Delete</a></td>
</tr>
</table>
</div>
</div>
<div>
<h2>
Edit Data Source
</h2>
<form id="datasource_form" novalidate>
<p>
<label for="id_name">Name:</label>
<input id="id_name" size="100" type="text" ng-model="datasource.name"/>
</p>
<p>
<label for="id_source_type">Type:</label>
<select id="id_source_type" ng-model="datasource.source_type" ng-options="type.0 as type.1 for type in creation_information.types" ng-disabled="readonly" ng-change="clearTypeParameters()">
<option value="">---------</option>
</select>
</p>
<p>
<label for="id_refresh_interval">Refresh Interval:</label>
<select id="id_refresh_interval" ng-model="datasource.refresh_interval" ng-options="interval.0 as interval.1 for interval in creation_information.refresh_intervals" ng-disabled="readonly">
<option value="">---------</option>
</select>
</p>
<p>
<label for="id_policy">Data Policy:</label>
<select id="id_policy" ng-model="datasource.data_policy" ng-options="policy.0 as policy.1 for policy in creation_information.data_policies" ng-disabled="readonly">
<option value="">---------</option>
</select>
</p>
<p>
<label for="id_datamart">Database:</label>
<select id="id_datamart" ng-model="datasource.database.id" ng-options="dm.id as dm.name for dm in creation_information.databases" ng-disabled="readonly">
</select>
</p>
<span ng-repeat="parameter in creation_information.available_datasources[datasource.source_type]">
<p ng-if="!parameter.parent || datasource.parameters[parameter.parent]">
<label for="id_database">{{parameter.name}}</label>
<span ng-if="parameter.values.length > 0">
<label class="Select">
<select ng-model="datasource.parameters[parameter.name]" ng-options="f.key as f.name for f in field_for(table_filter, $index).values" ng-disabled="{{readonly}}">
</select>
</label>
</span>
<span ng-if="parameter.values.length == 0">
<input class="input" size="30" type="{{parameter.type}}" ng-model="datasource.parameters[parameter.name]" ng-readonly="{{readonly}}"/ >
</span>
</p>
</span>
<table width="100%">
<tr>
<td>
<a class="Button mx1" href="#" ng-click="save(datasource)">Save</a>
</td>
<td style="text-align: right">
</td>
</tr>
</table>
</form>
</div>
<div class=" clearfix">
<div class="float-right">
<div class="mx2 inline-block">
<a class="Button mx1" href="#" ng-click="ingest()">Ingest Now</a>
<a class="Button mx1" href="#" ng-if="datasource.source_type=='log'" ng-click="reingest()">Re-Ingest All</a>
<a class="Button" cv-org-href="/admin/datasources/{{datasource.id}}/modify">Edit</a>
</div>
</div>
<h2>Data Source: {{datasource.name}}</h2>
</div>
<div class="page-header row">
<div class="TableWrapper py4">
<table class="Table">
<thead>
<tr>
</tr>
</thead>
<tbody>
<tr>
<td>ID</td>
<td>{{datasource.id}}</td>
</tr>
<tr>
<td>Name</td>
<td>{{datasource.name}}</td>
</tr>
<tr>
<td>Type</td>
<td>{{datasource.source_type}}</td>
</tr>
<tr>
<td>Database</td>
<td>{{datasource.database.name}}</td>
</tr
>
<tr>
<td>Refresh Interval</td>
<td>{{datasource.refresh_interval}}</td>
</tr>
<tr>
<td>Data Policy</td>
<td>{{datasource.data_policy}}</td>
</tr>
<tr ng-repeat="(k, v) in datasource.parameters">
<td>{{k}}</td>
<td>{{v}}</td>
</tr>
</tbody>
</table>
</div>
<div class="page-info">
<h2 class="page-title">Ingestions</h2>
</div>
</div>
<div class="page-content row">
<div class="TableWrapper py4">
<table class="Table">
<thead>
<tr>
<th>ID</th>
<th>Source</th>
<th># Tables</th>
<th># Rows</th>
<th>Status</th>
<th>Database</th>
<th ng-if="isTimestampedDataSource">File Timestamp</th>
<th>downloading_start</th>
<th>downloading_end</th>
<th>transformation_start</th>
<th>transformation_end</th>
<th>insertion_start</th>
<th>insertion_end</th>
<th>sync_start</th>
<th>sync_end</th>
</tr>
</thead>
<tr ng-if="!ingestions">
<td colspan="7" style="text-align:center;"><h3>Loading ...</h3></td>
</tr>
<tr ng-repeat="ingestion in ingestions">
<td><a cv-org-href="/admin/datasource_ingestion/{{ingestion.id}}">{{ingestion.id}}</a></td>
<td><a cv-org-href="/admin/datasources/{{ingestion.source}}">{{ingestion.source}}</a></td>
<td>{{ingestion.num_tables}}</td>
<td>{{ingestion.num_rows}}</td>
<td>{{ingestion.status}}</td>
<td>{{ingestion.database_name}}</td>
<td ng-if="isTimestampedDataSource">{{ingestion.write_time}}</td>
<td>{{ingestion.downloading_start}}</td>
<td>{{ingestion.downloading_end}}</td>
<td>{{ingestion.transformation_start}}</td>
<td>{{ingestion.transformation_end}}</td>
<td>{{ingestion.insertion_start}}</td>
<td>{{ingestion.insertion_end}}</td>
<td>{{ingestion.sync_start}}</td>
<td>{{ingestion.sync_end}}</td>
</tr>
</table>
</div>
</div>
<div class="float-right">
<a href="#" class="Button Button--primary" ng-if="page > 1" ng-click="setPage(page-1)">Prev</a>
<a href="#" class="Button Button--primary" ng-click="setPage(page+1)">Next</a>
</div>
......@@ -3,8 +3,8 @@
var EmailReportServices = angular.module('corvusadmin.emailreport.services', ['ngResource', 'ngCookies']);
EmailReportServices.service('EmailReportUtils', function () {
this.weekdayFullName = function (abbr) {
EmailReportServices.service('EmailReportUtils', function() {
this.weekdayFullName = function(abbr) {
var days_of_week = {
"sun": "Sunday",
"mon": "Monday",
......@@ -22,7 +22,7 @@ EmailReportServices.service('EmailReportUtils', function () {
}
};
this.humanReadableSchedule = function (schedule) {
this.humanReadableSchedule = function(schedule) {
// takes in a dictionary representation of a 'schedule' from an EmailReport
// and provides a human readable representation of what's going to take place
......@@ -34,7 +34,7 @@ EmailReportServices.service('EmailReportUtils', function () {
var msg_day = "";
var cnt = 0;
var selected = [];
Object.keys(schedule.days_of_week).forEach(function (key) {
Object.keys(schedule.days_of_week).forEach(function(key) {
if (schedule.days_of_week[key]) {
cnt++;
selected.push(key);
......@@ -44,14 +44,14 @@ EmailReportServices.service('EmailReportUtils', function () {
if (cnt === 0) {
return "You're report will not run because you haven't selected any days to run it on.";
} else if (cnt === 1) {
msg_day = "on "+this.weekdayFullName(selected[0])+"s";
msg_day = "on " + this.weekdayFullName(selected[0]) + "s";
} else if (cnt === 2) {
msg_day = "on "+this.weekdayFullName(selected[0])+"s and "+this.weekdayFullName(selected[1])+"s";
msg_day = "on " + this.weekdayFullName(selected[0]) + "s and " + this.weekdayFullName(selected[1]) + "s";
} else if (cnt === 7) {
msg_day = "every day";
} else {
// last case is 3-6 days selected
msg_day = cnt+" days a week";
msg_day = cnt + " days a week";
}
msg = msg + msg_day;
......@@ -62,9 +62,9 @@ EmailReportServices.service('EmailReportUtils', function () {
var msg_time = "";
if (_.contains(['morning', 'evening', 'afternoon'], schedule.time_of_day)) {
msg_time = " in the "+schedule.time_of_day;
msg_time = " in the " + schedule.time_of_day;
} else {
msg_time = " at "+schedule.time_of_day;
msg_time = " at " + schedule.time_of_day;
}
msg = msg + msg_time;
......@@ -72,7 +72,7 @@ EmailReportServices.service('EmailReportUtils', function () {
// lastly, add on our timezone
if (schedule.timezone) {
msg = msg + " ("+schedule.timezone+")";
msg = msg + " (" + schedule.timezone + ")";
}
return msg;
......@@ -82,56 +82,68 @@ EmailReportServices.service('EmailReportUtils', function () {
EmailReportServices.factory('EmailReport', ['$resource', '$cookies', function($resource, $cookies) {
return $resource('/api/emailreport/:reportId', {}, {
form_input: {
url:'/api/emailreport/form_input?org=:orgId',
method:'GET',
url: '/api/emailreport/form_input?org=:orgId',
method: 'GET',
},
list: {
url:'/api/emailreport/?org=:orgId&f=:filterMode',
method:'GET',
isArray:true
url: '/api/emailreport/?org=:orgId&f=:filterMode',
method: 'GET',
isArray: true
},
create: {
url:'/api/emailreport/',
method:'POST',
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
url: '/api/emailreport/',
method: 'POST',
headers: {
'X-CSRFToken': function() {
return $cookies.csrftoken;
}
}
},
get: {
method:'GET',
params:{reportId:'@reportId'}
method: 'GET',
params: {
reportId: '@reportId'
}
},
update: {
method:'PUT',
params:{reportId:'@id'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
method: 'PUT',
params: {
reportId: '@id'
},
headers: {
'X-CSRFToken': function() {
return $cookies.csrftoken;
}
}
},
delete: {
method:'DELETE',
params:{reportId:'@reportId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
method: 'DELETE',
params: {
reportId: '@reportId'
},
headers: {
'X-CSRFToken': function() {
return $cookies.csrftoken;
}
}
},
execute: {
method:'POST',
params:{reportId:'@reportId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
method: 'POST',
params: {
reportId: '@reportId'
},
headers: {
'X-CSRFToken': function() {
return $cookies.csrftoken;
}
}
},
recent_execs: {
url:'/url/emailreport/@reportId/executions',
method:'GET',
params:{reportId:'@reportId'}
}
});
}]);
EmailReportServices.factory('EmailReportExec', ['$resource', '$cookies', function($resource, $cookies) {
return $resource('/api/etl/jobexec/:execId', {}, {
list: {
url:'/api/etl/jobexec',
method:'GET',
isArray:true
},
get: {
method:'GET',
params:{execId:'@execId'}
url: '/url/emailreport/@reportId/executions',
method: 'GET',
params: {
reportId: '@reportId'
}
}
});
}]);
}]);
\ No newline at end of file
'use strict';
/*global _*/
var EtlControllers = angular.module('corvusadmin.etl.controllers', []);
EtlControllers.controller('EtlJobexecList', ['$scope', '$routeParams', '$interval', 'EtlJobExec', function($scope, $routeParams, $interval, EtlJobExec) {
// $scope.jobexecs
$scope.filterMode = 'all';
$scope.sortMode = 'lastexec';
var jobMonitor;
$scope.filter = function(mode) {
$scope.filterMode = mode;
$scope.$watch('currentOrg', function (org) {
if (!org) return;
EtlJobExec.list({
'orgId': org.id,
'filterMode': mode
}, function(result) {
$scope.jobexecs = result;
$scope.sort();
});
});
};
$scope.sort = function() {
if ('table_name' == $scope.sortMode) {
$scope.jobexecs.sort(function(a, b) {
return a.details.table_name.localeCompare(b.details.table_name);
});
} else if ('owner' == $scope.sortMode) {
$scope.jobexecs.sort(function(a, b) {
return a.job.creator.email.localeCompare(b.job.creator.email);
});
} else if ('name' == $scope.sortMode) {
$scope.jobexecs.sort(function(a, b) {
return a.job.name.localeCompare(b.job.name);
});
} else {
// default mode is by last exec descending
$scope.jobexecs.sort(function(a, b) {
a = new Date(a.created_at);
b = new Date(b.created_at);
return a > b ? -1 : a < b ? 1 : 0;
});
}
};
$scope.canceljobMonitor = function() {
if (angular.isDefined(jobMonitor)) {
$interval.cancel(jobMonitor);
jobMonitor = undefined;
}
};
// determine the appropriate filter to start with
$scope.filter($scope.filterMode);
$scope.$on('$destroy', function() {
$scope.canceljobMonitor();
});
// start an interval which will refresh our listing periodically
jobMonitor = $interval(function() {
$scope.filter($scope.filterMode);
}, 30000);
}]);
EtlControllers.controller('EtlJobList', ['$scope', '$routeParams', '$location', 'EtlJob', function($scope, $routeParams, $location, EtlJob) {
// $scope.jobs
$scope.filterMode = 'all';
$scope.sortMode = 'name';
$scope.filter = function(mode) {
$scope.filterMode = mode;
$scope.$watch('currentOrg', function (org) {
if (!org) return;
EtlJob.list({
'orgId': org.id,
'filterMode': mode
}, function(result) {
$scope.jobs = result;
$scope.sort();
});
});
};
$scope.sort = function() {
if ('date' == $scope.sortMode) {
$scope.jobs.sort(function(a, b) {
a = new Date(a.updated_at);
b = new Date(b.updated_at);
return a > b ? -1 : a < b ? 1 : 0;
});
} else if ('org' == $scope.sortMode) {
$scope.jobs.sort(function(a, b) {
return a.organization.name.localeCompare(b.organization.name);
});
} else if ('owner' == $scope.sortMode) {
$scope.jobs.sort(function(a, b) {
return a.creator.email.localeCompare(b.creator.email);
});
} else {
$scope.jobs.sort(function(a, b) {
return a.name.localeCompare(b.name);
});
}
};
$scope.executeJob = function(jobId) {
EtlJob.execute({
'jobId': jobId
}, function(result) {
// send the user over to the job exec listing page
$location.path('/' + $scope.currentOrg.slug + '/admin/etl/jobexec/');
}, function(error){
console.log('error', error);
});
};
$scope.deleteJob = function(jobId) {
if ($scope.jobs) {
EtlJob.delete({
'jobId': jobId
}, function(result) {
$scope.jobs = _.filter($scope.jobs, function(job){
return job.id != jobId;
});
$scope.searchFilter = undefined;
}, function(error){
console.log(error);
$scope.alertError('failed to remove job');
});
}
};
// start by showing all jobs
$scope.filter($scope.filterMode);
}]);
EtlControllers.controller('EtlJobDetail', ['$scope', '$routeParams', '$location', 'EtlJob', function($scope, $routeParams, $location, EtlJob) {
// $scope.job
// $scope.success_message
// $scope.error_message
$scope.save = function(jobDetail) {
$scope.clearStatus();
if ($scope.job.id) {
// if there is already an ID associated with the job then we are updating
EtlJob.update(jobDetail, function(result) {
if (result && !result.error) {
$scope.job = result;
// alert
$scope.success_message = 'EtlJob saved successfully!';
} else {
console.log(result);
$scope.error_message = 'Failed saving EtlJob!';
}
});
} else {
// otherwise we are creating a new job
jobDetail.org = $scope.currentOrg.id;
EtlJob.create(jobDetail, function(result) {
if (result && !result.error) {
$scope.job = result;
// move the user over the the actual page for the new job
$location.path('/' + $scope.currentOrg.slug + '/admin/etl/job/' + result.id);
} else {
console.log(result);
$scope.error_message = 'Failed saving EtlJob!';
}
});
}
};
$scope.executeJob = function(jobId) {
EtlJob.execute({
'jobId': jobId
}, function(result) {
if (result && !result.error) {
// send user to listing page with a focus on our new job execution
$location.path('/' + $scope.currentOrg.slug + '/admin/etl/jobexec/');
} else {
console.log('error', result);
}
});
};
$scope.addStep = function() {
if ($scope.job) {
$scope.job.details.steps.push({"type":"sql", "sql":{"query":""}});
}
};
$scope.removeStep = function() {
if ($scope.job) {
$scope.job.details.steps.splice($scope.job.details.steps.length - 1, 1);
// strange angularism. this needs to be here because of the delete-confirm dialog
// somehow if we don't force digest then the UI won't be notified about our change
$scope.$digest();
}
};
$scope.clearStatus = function() {
$scope.success_message = undefined;
$scope.error_message = undefined;
};
EtlJob.form_input(function (form_input) {
$scope.form_input = form_input;
}, function (error) {
console.log('error getting EtlJob form_input', error);
});
if ($routeParams.jobId) {
// fetch the Job data
EtlJob.get({
'jobId': $routeParams.jobId
}, function(result) {
$scope.job = result;
}, function(error) {
console.log(error);
if (error.status == 404) {
$location.path('/');
}
});
} else {
// user must be creating a new Job, so start them off with a basic template
$scope.job = {
"name": "",
"details": {
"type": "sql_table",
"database": undefined,
"table_name": "",
"steps": [
{
"type": "sql",
"sql": {
"query": ""
}
}
]
}
};
}
}]);
'use strict';
var ETL = angular.module('corvusadmin.etl', [
'corvusadmin.etl.controllers',
'corvusadmin.etl.services'
]);
ETL.config(['$routeProvider', function ($routeProvider) {
$routeProvider.when('/:orgSlug/admin/etl/jobexec/', {templateUrl: '/app/admin/etl/partials/etl_jobexec_list.html', controller: 'EtlJobexecList'});
$routeProvider.when('/:orgSlug/admin/etl/job/', {templateUrl: '/app/admin/etl/partials/etl_job_list.html', controller: 'EtlJobList'});
$routeProvider.when('/:orgSlug/admin/etl/job/create', {templateUrl: '/app/admin/etl/partials/etl_job_detail.html', controller: 'EtlJobDetail'});
$routeProvider.when('/:orgSlug/admin/etl/job/:jobId', {templateUrl: '/app/admin/etl/partials/etl_job_detail.html', controller: 'EtlJobDetail'});
$routeProvider.otherwise({redirectTo: '/:orgSlug/admin/'});
}]);
'use strict';
// ETL Services
var EtlServices = angular.module('corvusadmin.etl.services', ['ngResource', 'ngCookies']);
EtlServices.factory('EtlJob', ['$resource', '$cookies', function($resource, $cookies) {
return $resource('/api/etl/job/:jobId', {}, {
form_input: {
url:'/api/etl/job/form_input',
method:'GET',
},
list: {
url:'/api/etl/job/?org=:orgId&f=:filterMode',
method:'GET',
isArray:true
},
create: {
url:'/api/etl/job',
method:'POST',
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
get: {
method:'GET',
params:{jobId:'@jobId'}
},
update: {
method:'PUT',
params:{jobId:'@id'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
delete: {
method:'DELETE',
params:{jobId:'@jobId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
execute: {
method:'POST',
params:{jobId:'@jobId'},
headers: {'X-CSRFToken': function() { return $cookies.csrftoken; }},
},
job_execs: {
url:'/url/etl/job/@jobId/execs',
method:'GET',
params:{jobId:'@jobId'}
}
});
}]);
EtlServices.factory('EtlJobExec', ['$resource', '$cookies', function($resource, $cookies) {
return $resource('/api/etl/jobexec/:execId', {}, {
list: {
url:'/api/etl/jobexec?org=:orgId',
method:'GET',
isArray:true
},
get: {
method:'GET',
params:{execId:'@execId'}
}
});
}]);
<div class="wrapper">
<form class="Form" novalidate>
<div class="py2 border-bottom">
<h2>Job Details</h2>
<div class="py1 float-right">
<div>
<label for="id_mode">Mode:</label>
<select class="block" id="id_mode" ng-init="job.mode = utils.etljob_modes[0].id" ng-model="job.mode" ng-options="mode.id as mode.name for mode in utils.etljob_modes">
</select>
</div>
<div class="py2">
<label for="id_perms">Permissions:</label>
<select class="block" id="id_perms" ng-init="job.public_perms = utils.perms[2].id" ng-model="job.public_perms" ng-options="perm.id as perm.name for perm in utils.perms">
</select>
</div>
</div>
<div class="py1">
<label for="id_name">Job Name:</label>
<input class="input block" id="id_name" size="80" maxlength="255" type="text" ng-model="job.name"/>
</div>
<div class="py1 clearfix">
<label for="id_database">Database:</label>
<select class="block" id="id_database" ng-model="job.database" ng-options="dm.id as dm.name for dm in form_input['databases']">
</select>
</div>
<div class="py1">
<label for="id_sql_table">Output SQL Table:</label>
<input class="input block" id="id_sql_table" size="80" maxlength="255" type="text" ng-model="job.details.table_name"/>
</div>
<div class="">
<label>Connection Timezone:</label>
<select class="block" ng-model="job.details.timezone" ng-options="tz for tz in form_input['timezones']">
<option value="">----- (database tz)</option>
</select>
</div>
</div>
<div class="py2">
<h2>Job Steps</h2>
<div class="py1" ng-repeat="step in job.details.steps">
<label for="id_sql">Step #{{$index}}:</label>
<textarea class="input block full" id="id_sql" rows="6" ng-model="step.sql.query"></textarea>
</div>
<p>
Hint: You can refer to the output of any given step as a SQL Table by using the ${table_&lt;step_number&gt;} syntax. Ex: ${table_0} refers to the output of the first step of your job.
</p>
<div class="py1">
<a class="Button" ng-click="addStep()">Add Step</a>
<a class="Button" ng-click="removeStep()" delete-confirm>Remove Step</a>
</div>
</div>
<div class="my1 py2 border-top">
<div class="float-right">
<input class="Button" type="button" value="Execute Now" ng-if="job.id" ng-click="executeJob(job.id)"/>
</div>
<input class="Button Button--primary" type="button" value="Save" ng-click="save(job)"/>
<div class="inline-block" ng-cloak>
<span class="px2 text-success" ng-if="success_message" cv-delayed-call="clearStatus()">{{success_message}}</span>
<span class="px2 text-error" ng-if="error_message" cv-delayed-call="clearStatus()">{{error_message}}</span>
<alert class="alert" ng-repeat="alert in alerts" type="{{alert.type}}" close="closeAlert($index)" ng-animate=" 'animate' " cv-delayed-call="closeAlert($index)">{{alert.msg}}</alert>
</div>
</div>
</form>
</div>
<div class="wrapper">
<div class="py2 border-bottom">
<div class="float-right">
<a class="Button Button--primary" cv-org-href="/admin/etl/job/create">Create ETL Job</a>
</div>
<h2>ETL Job List</h2>
</div>
<div>
<div class="List-filters List-section clearfix">
<div class="List-filterCategories float-left">
<span class="List-filterLabel float-left">filter:</span>
<div class="Button-group">
<a class="Button" ng-class="{ 'Button--selected' : filterMode == 'all' }" ng-click="filter('all')">all</a>
<a class="Button" ng-class="{ 'Button--selected' : filterMode == 'mine' }" ng-click="filter('mine')">mine</a>
</div>
</div>
<div class="float-left">
<input class="mx3 input" type="text" ng-model="searchFilter" placeholder="Search executions ...">
</div>
<div class="float-right">
<span>Sort by:</span>
<select ng-init="sortMode = 'name'" ng-model="sortMode" ng-change="sort()">
<option value="name">Name (default)</option>
<option value="owner">Owner</option>
<option value="date">Last Updated</option>
</select>
</div>
</div>
<div ng-if="!jobs">
<h3>Loading ...</h3>
</div>
<ul>
<li class="py2 border-bottom" ng-repeat="job in jobs | filter:searchFilter">
<div class="my1 float-right">
<a class="Button mx1" ng-click="executeJob(job.id)">Execute</a>
<a class="Button mx1" ng-click="deleteJob(job.id)" delete-confirm>Delete</a>
</div>
<div class="my1 px2 float-right">
<span>{{job.updated_at | date : 'MMMM d, y, hh:mm a'}}</span>
</div>
<h3>
<a class="text-normal link" cv-org-href="/admin/etl/job/{{job.id}}">{{job.name}}</a>
</h3>
<span>{{job.details.table_name}}</span>
<span>[{{job.creator.common_name}}]</span>
</li>
</ul>
</div>
</div>
<div class="wrapper">
<div class="py2 border-bottom">
<h2>ETL Job Executions</h2>
</div>
<div>
<div class="List-filters List-section clearfix">
<div class="List-filterCategories float-left">
<span class="List-filterLabel float-left">filter:</span>
<div class="Button-group">
<a class="Button" ng-class="{ 'Button--selected' : filterMode == 'all' }" ng-click="filter('all')">all</a>
<a class="Button" ng-class="{ 'Button--selected' : filterMode == 'mine' }" ng-click="filter('mine')">mine</a>
</div>
</div>
<div class="float-left">
<input class="mx3 input" type="text" ng-model="searchFilter" placeholder="Search executions ...">
</div>
<div class="float-right">
<span>Sort by:</span>
<select ng-init="sortMode = 'lastexec'" ng-model="sortMode" ng-change="sort()">
<option value="lastexec">Last Execution (default)</option>
<option value="name">Name</option>
<option value="owner">Owner</option>
</select>
</div>
</div>
<div ng-if="!jobexecs">
<h3 class="text-normal text-centered">Loading ...</h3>
</div>
<ul>
<li class="py2 border-bottom" ng-repeat="jobexec in jobexecs | filter:searchFilter">
<div class="my1 float-right">
<span class="mx1">{{jobexec.created_at | date : 'MMMM d, y, hh:mm a'}}</span>
<span class="mx1">
<span class="text-success" ng-if="jobexec.status == 'completed'">{{jobexec.status}}</span>
<span class="text-error" ng-if="jobexec.status == 'failed'">{{jobexec.status}}</span>
<span ng-if="jobexec.status == 'running'">{{jobexec.status}}</span>
</span>
<span class="mx1">{{jobexec.steps_status.steps.length | isempty: '0'}} / {{jobexec.details.steps.length}}</span>
<a ng-click="execDetail = !execDetail">
<span ng-if="!execDetail">more</span>
<span ng-if="execDetail">less</span>
</a>
</div>
<h3>
<a class="text-normal link" cv-org-href="/admin/etl/job/{{jobexec.job.id}}">{{jobexec.job.name}}</a>
</h3>
<span>{{jobexec.details.table_name}}</span>
<span>[{{jobexec.job.creator.common_name}}]</span>
<ul class="my1 bordered rounded ng-hide" ng-show="execDetail">
<li class="py2 px2 border-bottom clearfix" ng-repeat="step in jobexec.steps_status.steps">
<h5 class="float-left">Step #{{$index}} ({{step.end - step.start}}ms)</h5>
<div class="float-right">
<span class="mx1"></span>
<span class="mx1">{{step.start | date : 'MM-d-y HH:mm:ss.sss'}}</span>
<span class="mx1">{{step.end | date : 'MM-d-y HH:mm:ss.sss'}}</span>
<span class="mx1">
<span class="text-success" ng-if="step.status == 'completed'">{{step.status}}</span>
<span class="text-error" ng-if="step.status == 'failed'">{{step.status}}</span>
<span ng-if="step.status == 'running'">{{step.status}}</span>
</span>
</div>
<div class="inline-block" ng-if="step.status == 'failed'">
<pre>{{step.error}}</pre>
</div>
</li>
</ul>
</li>
</ul>
</div>
</div>
......@@ -19,7 +19,6 @@ var Corvus = angular.module('corvus', [
'corvus.components',
'corvus.card',
'corvus.dashboard',
'corvus.etl',
'corvus.explore',
'corvus.operator', // this is a short term hack
'corvus.reserve',
......@@ -65,4 +64,4 @@ if (document.location.hostname != "localhost") {
}).run(function(Angularytics) {
Angularytics.init();
});
}
}
\ No newline at end of file
......@@ -42,7 +42,7 @@ DashboardDirectives.directive('cvAddToDashboardModal', ['CorvusCore', 'Dashboard
var existingDashboardsById = {};
Dashboard.list({
'orgId': $scope.card.organization.id,
'orgId': $scope.card.organization_id,
'filterMode': 'all'
}, function(result) {
if (result && !result.error) {
......@@ -106,4 +106,4 @@ DashboardDirectives.directive('cvAddToDashboardModal', ['CorvusCore', 'Dashboard
card: '='
}
};
}]);
}]);
\ No newline at end of file
'use strict';
// ETL Controllers
var ETLControllers = angular.module('corvus.etl.controllers', []);
ETLControllers.controller('ETLIngestionList', ['$scope', '$routeParams', '$location', 'ETL', function($scope, $routeParams, $location, ETL) {
$scope.setPage = function(page) {
// what type of ingestions are we displaying?
var etltype = $location.path().split('/')[4];
ETL.ingestion_list({
'type': etltype,
'dbId': $routeParams.db,
'page': page
}, function(result) {
$scope.ingestions = result;
$scope.page = page;
}, function (error) {
console.log(error);
$location.path('/');
});
};
$scope.setPage(1);
}]);
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment