Add source stat model

Save source stat information

Update sources who have corrected their earliestDatestamp

Catch and log all source exceptions

Author: laurenbarker

project/settings.py

@@ -327,6 +327,12 @@
         'schedule': crontab(hour=23, minute=30),
         'args': (1, 'elasticsearch'),
     },
+    # Executes daily at 11:30 P.M
+    'source-stat-task': {
+        'task': 'share.tasks.SourceStatTask',
+        'schedule': crontab(hour=23, minute=30),
+        'args': (),
+    },
 }
 
 CELERY_TASK_SERIALIZER = 'json'

share/migrations/0034_auto_20170601_1703.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.1 on 2017-06-01 17:03
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import share.models.ingest
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('share', '0033_merge_20170509_1710'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SourceStat',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('is_deleted', models.BooleanField(default=False)),
+                ('date_created', models.DateTimeField(auto_now_add=True)),
+                ('response_status_code', models.SmallIntegerField(blank=True, null=True)),
+                ('response_elapsed_time', models.FloatField(blank=True, null=True)),
+                ('response_exception', models.TextField(blank=True, null=True)),
+                ('earliest_datestamp_config', models.DateField(blank=True, null=True)),
+                ('base_url_config', models.TextField()),
+                ('admin_note', models.TextField(blank=True)),
+            ],
+            managers=[
+                ('objects', share.models.ingest.NaturalKeyManager('config.label')),
+            ],
+        ),
+        migrations.CreateModel(
+            name='SourceStatOAI',
+            fields=[
+                ('sourcestat_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='share.SourceStat')),
+                ('earliest_datestamp_source', models.DateField(blank=True, null=True)),
+                ('earliest_datestamps_match', models.BooleanField(default=False)),
+                ('base_url_source', models.TextField(blank=True, null=True)),
+                ('base_urls_match', models.BooleanField(default=False)),
+            ],
+            bases=('share.sourcestat',),
+        ),
+        migrations.AddField(
+            model_name='sourcestat',
+            name='config',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='share.SourceConfig'),
+        ),
+    ]

share/models/__init__.py

@@ -10,6 +10,7 @@
 from share.models.registration import *  # noqa
 from share.models.identifiers import *  # noqa
 from share.models.relations import *  # noqa
-from share.models.banner import * # noqa
-from share.models.ingest import * # noqa
-from share.models.logs import * # noqa
+from share.models.banner import *  # noqa
+from share.models.ingest import *  # noqa
+from share.models.logs import *  # noqa
+from share.models.sources import *  # noqa

share/models/sources.py

@@ -0,0 +1,40 @@
+import logging
+
+from django.db import models
+
+from share.models.ingest import SourceConfig
+from share.models.ingest import NaturalKeyManager
+
+logger = logging.getLogger(__name__)
+__all__ = ('SourceStat', 'SourceStatOAI',)
+
+
+class SourceStat(models.Model):
+    config = models.ForeignKey(SourceConfig, on_delete=models.CASCADE)
+    is_deleted = models.BooleanField(default=False)
+    date_created = models.DateTimeField(auto_now_add=True)
+    response_status_code = models.SmallIntegerField(blank=True, null=True)
+    response_elapsed_time = models.FloatField(blank=True, null=True)
+    response_exception = models.TextField(blank=True, null=True)
+    earliest_datestamp_config = models.DateField(blank=True, null=True)
+    base_url_config = models.TextField()
+    admin_note = models.TextField(blank=True)
+
+    objects = NaturalKeyManager('config.label')
+
+    def natural_key(self):
+        return (self.config.label,)
+
+    def __repr__(self):
+        return '<{}({}, {})>'.format(self.__class__.__name__, self.pk, self.config.label)
+
+    def __str__(self):
+        return '{}: {}'.format(self.config.source.long_title, self.config.label)
+
+
+class SourceStatOAI(SourceStat):
+    earliest_datestamp_source = models.DateField(blank=True, null=True)
+    earliest_datestamps_match = models.BooleanField(default=False)
+
+    base_url_source = models.TextField(blank=True, null=True)
+    base_urls_match = models.BooleanField(default=False)

share/sources/be.ghent/source.yaml

@@ -1,7 +1,7 @@
 configs:
 - base_url: https://biblio.ugent.be/oai
   disabled: false
-  earliest_date: null  # earliestDatestamp is earliest published
+  earliest_date: 2016-12-14T15:38:10Z
   harvester: oai
   harvester_kwargs: {metadata_prefix: mods}
   label: be.ghent.mods

share/sources/ch.cern/source.yaml

@@ -1,7 +1,7 @@
 configs:
 - base_url: http://cds.cern.ch/oai2d
   disabled: false
-  earliest_date: null  # earliestDatestamp is earliest published
+  earliest_date: 2003-06-02T08:06:23Z
   harvester: oai
   harvester_kwargs: {metadata_prefix: oai_dc}
   label: ch.cern

share/sources/edu.udc/source.yaml

@@ -1,7 +1,7 @@
 configs:
 - base_url: http://conservancy.umn.edu/oai/request
   disabled: false
-  earliest_date: null  # earliestDatestamp is earliest published
+  earliest_date: 2007-02-19T20:04:59Z
   harvester: oai
   harvester_kwargs: {metadata_prefix: mods}
   label: edu.udc.mods

share/sources/es.csic/source.yaml

@@ -2,7 +2,7 @@
 configs:
 - base_url: http://digital.csic.es/dspace-oai/request
   disabled: false
-  earliest_date: null  # earliestDatestamp is earliest published
+  earliest_date: 2007-05-07T22:00:00Z
   harvester: oai
   harvester_kwargs: {metadata_prefix: mods}
   label: es.csic.mods

share/tasks.py

@@ -8,6 +8,8 @@
 import pendulum
 import celery
 import requests
+from furl import furl
+from lxml import etree
 
 from django.apps import apps
 from django.conf import settings
@@ -21,6 +23,7 @@
 from share.harvest.exceptions import HarvesterConcurrencyError, HarvesterDisabledError
 from share.models import HarvestLog
 from share.models import Source
+from share.models import SourceStat, SourceStatOAI
 from share.models import RawDatum, NormalizedData, ChangeSet, CeleryTask, CeleryProviderTask, ShareUser, SourceConfig
 
 
@@ -407,3 +410,159 @@ def do_run(self, last_run=None, **kwargs):
         bot = self.config.get_bot(self.started_by, last_run=last_run, **kwargs)
         logger.info('Running bot %s. Started by %s', bot, self.started_by)
         bot.run()
+
+
+NAMESPACES = {
+    'dc': 'http://purl.org/dc/elements/1.1/',
+    'ns0': 'http://www.openarchives.org/OAI/2.0/',
+    'oai_dc': 'http://www.openarchives.org/OAI/2.0/oai_dc/',
+}
+
+
+def get_field_from_identify(response, field):
+    parsed = etree.fromstring(response.content, parser=etree.XMLParser(recover=True))
+    return parsed.xpath('//ns0:Identify/ns0:{}'.format(field), namespaces=NAMESPACES)[0].text
+
+
+def assert_no_exception(url, timeout=15.0):
+    try:
+        r = requests.get(url, timeout=timeout)
+    # except all exception and log
+    except Exception as e:
+        logger.warning('Exception received from source: %s', e)
+        return (None, e)
+    return (r, None)
+
+
+@celery.task(bind=True)
+def get_oai_source_stats(self, *args, **kwargs):
+    '''
+        Known incorrect baseUrl:
+            'https://biblio.ugent.be/oai',  # homepage listed
+            'http://purr.purdue.edu/oaipmh',  # homepage listed
+            'https://www.ncbi.nlm.nih.gov/pmc/oai/oai.cgi',  # incorrect url listed
+            'https://mla.hcommons.org/deposits/oai/',  # incorrect url listed
+            'http://oai.repec.org',  # listed url redirects to this
+
+
+        Known incorrect earliestDatestamp (all emailed):
+            'edu.oaktrust.mods',  # returns the most recent datestamp
+            'edu.scholarsarchiveosu.mods',  # returns 0011-01-01
+            'edu.uwashington.mods',   # returns 2083-03-01
+            'gov.nodc',  # 1996-10-09
+            'org.philpapers',  # 1990-01-01T00:00:00Z
+            'org.ttu.mods',  # 1989-05-01T05:00:00Z
+            'edu.umich.mods',  # 1983-01-01T05:00:00Z
+            'edu.citeseerx',  # 1970-01-01
+            'br.pcurio',  # 1970-01-01
+            'edu.vtech.mods',  # 1900-02-02T05:00:00Z
+            'edu.icpsr',  # 01-01-1900
+            'pt.rcaap',  # 1900-01-01T00:00:00Z
+            'org.ucescholarship',  # 1885-05-01
+            'com.nature',  # 1869-11-04
+    '''
+
+    base_url_config = kwargs['base_url']
+    base_url_source = None
+    base_urls_match = False
+    config_id = kwargs['id']
+    earliest_datestamp_config = kwargs['earliest_date'] or None
+    earliest_datestamp_source = None
+    earliest_datestamps_match = False
+    response_exception = True
+    response_elapsed_time = 0
+    response_status_code = None
+
+    response, error = assert_no_exception(furl(base_url_config).set({'verb': 'Identify'}).url)
+    response_exception = error
+
+    if response:
+        base_url_source = get_field_from_identify(response, 'baseURL')
+        # ignores http vs https
+        if len(base_url_source.split('://', 1)) > 1:
+            base_urls_match = base_url_source.split('://', 1)[1] == base_url_config.split('://', 1)[1]
+        else:
+            logger.warning('Source baseURL is improper: %s', base_url_source)
+
+        earliest_datestamp_identify = get_field_from_identify(response, 'earliestDatestamp')
+        earliest_datestamp_source = pendulum.parse(earliest_datestamp_identify).to_date_string() if earliest_datestamp_identify else None
+        earliest_datestamps_match = earliest_datestamp_config == earliest_datestamp_source
+
+        response_elapsed_time = response.elapsed.total_seconds()
+        response_status_code = response.status_code
+
+    source_stat = SourceStatOAI.objects.create(
+        earliest_datestamp_source=earliest_datestamp_source,
+        earliest_datestamp_config=earliest_datestamp_config,
+        earliest_datestamps_match=earliest_datestamps_match,
+
+        base_url_source=base_url_source,
+        base_url_config=base_url_config,
+        base_urls_match=base_urls_match,
+
+        config_id=config_id,
+
+        response_status_code=response_status_code,
+        response_elapsed_time=response_elapsed_time,
+        response_exception=response_exception,
+    )
+    source_stat.save()
+
+
+@celery.task(bind=True)
+def get_source_stats(self, *args, **kwargs):
+    base_url_config = kwargs['base_url']
+    config_id = kwargs['id']
+    earliest_datestamp_config = kwargs['earliest_date'] or None
+    response_exception = True
+    response_elapsed_time = None
+    response_status_code = None
+
+    response, error = assert_no_exception(base_url_config)
+    response_exception = error
+
+    if response:
+        response_elapsed_time = response.elapsed.total_seconds()
+        response_status_code = response.status_code
+
+    source_stat = SourceStatOAI.objects.create(
+        earliest_datestamp_config=earliest_datestamp_config,
+        base_url_config=base_url_config,
+        config_id=config_id,
+        response_status_code=response_status_code,
+        response_elapsed_time=response_elapsed_time,
+        response_exception=response_exception,
+    )
+    source_stat.save()
+
+
+class SourceStatusTask(celery.Task):
+
+    def run(self):
+        self.get_oai_stats.apply_async()
+        self.get_non_oai_stats.apply_async()
+
+    @celery.task(bind=True)
+    def get_oai_stats(self):
+        oai_sourceconfigs = SourceConfig.objects.filter(
+            disabled=False,
+            base_url__isnull=False,
+            harvester__key='oai'
+        )
+
+        for config in oai_sourceconfigs.values():
+            config['earliest_date'] = config['earliest_date'].isoformat() if config['earliest_date'] else None
+            get_oai_source_stats.apply_async(kwargs=config)
+
+    @celery.task(bind=True)
+    def get_non_oai_stats(self):
+        non_oai_sourceconfigs = SourceConfig.objects.filter(
+            disabled=False,
+            base_url__isnull=False
+        ).exclude(
+            harvester__key='oai'
+        )
+
+        for config in non_oai_sourceconfigs.values():
+            config['earliest_date'] = config['earliest_date'].isoformat() if config['earliest_date'] else None
+            get_source_stats.apply_async(kwargs=config)