diff --git a/metadata b/metadata index 5cf9a8d..d96af33 160000 --- a/metadata +++ b/metadata @@ -1 +1 @@ -Subproject commit 5cf9a8d632726738ea9fa1e88621fa701b8652dd +Subproject commit d96af33d1f2ec0520e906360c21dc478acb8867b diff --git a/python/nistoar/midas/dap/service/mds3.py b/python/nistoar/midas/dap/service/mds3.py index 8d5ce9e..05cf957 100644 --- a/python/nistoar/midas/dap/service/mds3.py +++ b/python/nistoar/midas/dap/service/mds3.py @@ -17,7 +17,7 @@ Support for the web service frontend is provided via :py:class:`DAPApp` class, an implementation of the WSGI-based :ref:class:`~nistoar.pdr.publish.service.wsgi.SubApp`. """ -import os, re, pkg_resources +import os, re, pkg_resources, random, string from logging import Logger from collections import OrderedDict from collections.abc import Mapping, MutableMapping, Sequence, Callable @@ -92,6 +92,10 @@ RES_DELIM = const.RESONLY_EXTENSION.lstrip('/') EXTSCHPROP = "_extensionSchemas" +def random_id(prefix: str="", n: int=8): + r = ''.join(random.choices(string.ascii_uppercase + string.digits, k=n)) + return prefix+r + class DAPService(ProjectService): """ a project record request broker class for DAP records. @@ -481,13 +485,15 @@ def update_data(self, id, newdata, part=None, message="", _prec=None): """ return self._update_data(id, newdata, part, replace=False, message="", prec=_prec) - def clear_data(self, id, part=None, _prec=None): + def clear_data(self, id, part=None, message: str=None, _prec=None) -> bool: """ remove the stored data content of the record and reset it to its defaults. :param str id: the identifier for the record whose data should be cleared. :param stt part: the slash-delimited pointer to an internal data property. If provided, only that property will be cleared (either removed or set to an initial default). + :return: True the data was properly cleared; return False if ``part`` was specified but does not + yet exist in the data. :param ProjectRecord prec: the previously fetched and possibly updated record corresponding to ``id``. If this is not provided, the record will by fetched anew based on the ``id``. @@ -509,44 +515,61 @@ def clear_data(self, id, part=None, _prec=None): self._store.load_from(nerd) nerd = self._store.open(id) + provact = None try: if part: what = part if part == "authors": + if nerd.authors.count == 0: + return False nerd.authors.empty() elif part == "references": + if nerd.references.count == 0: + return False nerd.references.empty() elif part == FILE_DELIM: + if nerd.files.count == 0: + return False what = "files" nerd.files.empty() elif part == LINK_DELIM: + if nerd.nonfiles.count == 0: + return False what = "links" nerd.nonfiles.empty() elif part == "components": + if nerd.nonfiles.count == 0 and nerd.files.count == 0: + return False nerd.files.empty() nerd.nonfiles.empty() - elif part in "title rights disclaimer description".split(): + elif part in "title rights disclaimer description landingPage keyword".split(): resmd = nerd.get_res_data() + if part not in resmd: + return False del resmd[part] nerd.replace_res_data(resmd) else: - raise PartNotAccessible(_prec.id, path, "Clearing %s not allowed" % path) + raise PartNotAccessible(_prec.id, part, "Clearing %s not allowed" % part) - provact = Action(Action.PATCH, _prec.id, self.who, "clearing "+what) + if not message: + message = "clearing "+what + provact = Action(Action.PATCH, _prec.id, self.who, message) part = ("/"+part) if part.startswith("pdr:") else ("."+part) provact.add_subaction(Action(Action.DELETE, _prec.id+"#data"+part, self.who, - "clearing "+what)) - prec.status.act(self.STATUS_ACTION_CLEAR, "cleared "+what) + message)) + _prec.status.act(self.STATUS_ACTION_CLEAR, "cleared "+what) else: nerd.authors.empty() nerd.references.empty() nerd.files.empty() nerd.nonfiles.empty() - nerd.replace_res_data(self._new_data_for(_prec.id, prec.meta)) + nerd.replace_res_data(self._new_data_for(_prec.id, _prec.meta)) - provact = Action(Action.PATCH, _prec.id, self.who, "clearing all NERDm data") - prec.status.act(self.STATUS_ACTION_CLEAR, "cleared all NERDm data") + if not message: + message = "clearing all NERDm data" + provact = Action(Action.PATCH, _prec.id, self.who, message) + _prec.status.act(self.STATUS_ACTION_CLEAR, "cleared all NERDm data") except PartNotAccessible: # client request error; don't record action @@ -555,28 +578,30 @@ def clear_data(self, id, part=None, _prec=None): except Exception as ex: self.log.error("Failed to clear requested NERDm data, %s: %s", _prec.id, str(ex)) self.log.warning("Partial update is possible") - provact.message = "Failed to clear requested NERDm data" - self._record_action(provact) + if provact: + provact.message = "Failed to clear requested NERDm data" + self._record_action(provact) - prec.status.act(self.STATUS_ACTION_CLEAR, "Failed to clear NERDm data") - prec.set_state(status.EDIT) - prec.data = self._summarize(nerd) - self._try_save(prec) + _prec.status.act(self.STATUS_ACTION_CLEAR, "Failed to clear NERDm data") + _prec.set_state(status.EDIT) + _prec.data = self._summarize(nerd) + self._try_save(_prec) raise - prec.data = self._summarize(nerd) + _prec.data = self._summarize(nerd) if set_state: - prec.status.set_state(status.EDIT) + _prec.status.set_state(status.EDIT) try: - prec.save() + _prec.save() except Exception as ex: self.log.error("Failed to saved DBIO record, %s: %s", prec.id, str(ex)) raise finally: - self._record_action(provact) + self._record_action(provact) + return True def _update_data(self, id, newdata, part=None, prec=None, nerd=None, replace=False, message=""): @@ -777,6 +802,11 @@ def put_each_into(data, objlist): for fmd in files: nerd.files.set_file_at(fmd) + except InvalidUpdate as ex: + self.log.error("Invalid update to NERDm data not saved: %s: %s", prec.id, str(ex)) + if ex.errors: + self.log.error("Errors include:\n "+("\n ".join([str(e) for e in ex.errors]))) + raise except Exception as ex: provact.message = "Failed to save NERDm data update due to internal error" self.log.error("Failed to save NERDm metadata: "+str(ex)) @@ -1017,6 +1047,11 @@ def _update_part_nerd(self, path: str, prec: ProjectRecord, nerd: NERDResource, except PartNotAccessible: # client request error; don't record action raise + except InvalidUpdate as ex: + self.log.error("Invalid update to NERDm data not saved: %s: %s", prec.id, str(ex)) + if ex.errors: + self.log.error("Errors include:\n "+("\n ".join([str(e) for e in ex.errors]))) + raise except Exception as ex: self.log.error("Failed to save update to NERDm data, %s: %s", prec.id, str(ex)) self.log.warning("Partial update is possible") @@ -1697,7 +1732,8 @@ def _moderate_author(self, auth, doval=True): return auth _refprops = set(("@id _schema _extensionSchemas title abbrev proxyFor location label "+ - "description citation refType doi inPreparation vol volNumber pages publishYear").split()) + "description citation refType doi inPreparation vol volNumber pages "+ + "authors publishYear").split()) _reftypes = set(("IsDocumentedBy IsSupplementTo IsSupplementedBy IsCitedBy Cites IsReviewedBy "+ "IsReferencedBy References IsSourceOf IsDerivedFrom "+ "IsNewVersionOf IsPreviousVersionOf").split()) @@ -1737,6 +1773,11 @@ def _moderate_reference(self, ref, doval=True): except AttributeError as ex: raise InvalidUpdate("location or proxyFor: value is not a string", sys=self) from ex + # Penultimately, add an id if doesn't already have one + if not ref.get("@id"): + ref['@id'] = "REPLACE" + # ref['@id'] = random_id("ref:") + # Finally, validate (if requested) schemauri = NERDM_SCH_ID + "/definitions/BibliographicReference" if ref.get("_schema"): @@ -1747,6 +1788,8 @@ def _moderate_reference(self, ref, doval=True): if doval: self.validate_json(ref, schemauri) + if ref.get("@id") == "REPLACE": + del ref['@id'] return ref def _moderate_file(self, cmp, doval=True): @@ -1980,6 +2023,8 @@ def __init__(self, dbcli_factory: DBClientFactory, log: Logger, config: dict={}, class DAPProjectDataHandler(ProjectDataHandler): """ A :py:class:`~nistoar.midas.wsgi.project.ProjectDataHandler` specialized for editing NERDm records. + + Note that this implementation inherits its PUT, PATCH, and DELETE handling from its super-class. """ _allowed_post_paths = "authors references components".split() + [FILE_DELIM, LINK_DELIM] diff --git a/python/nistoar/midas/dap/service/validate.py b/python/nistoar/midas/dap/service/validate.py index ce15a9a..484fec5 100644 --- a/python/nistoar/midas/dap/service/validate.py +++ b/python/nistoar/midas/dap/service/validate.py @@ -1,12 +1,45 @@ """ validation utilities specialized for DAP editing """ +import re + from nistoar.nerdm.validate import * from nistoar.nerdm.constants import core_schema_base as CORE_SCHEMA_BASE +import nistoar.nerdm.utils as nerdm_utils PUB_SCHEMA_BASE = CORE_SCHEMA_BASE + "pub/" +RLS_SCHEMA_BASE = CORE_SCHEMA_BASE + "rls/" + +directives_by_uribase = { + CORE_SCHEMA_BASE: { + "derequire": [ "Resource", "Organization" ] + }, + PUB_SCHEMA_BASE: { + "derequire": [ "PublicDataResource", "Person" ] + }, + RLS_SCHEMA_BASE: { + "derequire": [ "ReleasedResource" ] + } +} +_verre = re.compile(r"/v\d.*$") class LenientSchemaLoader(ejs.SchemaLoader): + """ + this modifies the schema definitions on selected schemas to be more lenient for records + intended for use in the DAP Authoring API. + """ + def load_schema(self, uri): + out = super().load_schema(uri) + + if out.get("id", "").startswith(CORE_SCHEMA_BASE): + base = _verre.sub("/", out['id']) + directives = directives_by_uribase.get(base, {}) + nerdm_utils.loosen_schema(out, directives) + + return out + + +class OldLenientSchemaLoader(ejs.SchemaLoader): """ this modifies the schema definitions on selected schemas to be more lenient for records intended for use in the DAP Authoring API. diff --git a/python/nistoar/midas/dbio/project.py b/python/nistoar/midas/dbio/project.py index b7da9bf..7b00b81 100644 --- a/python/nistoar/midas/dbio/project.py +++ b/python/nistoar/midas/dbio/project.py @@ -142,6 +142,14 @@ def create_record(self, name, data=None, meta=None) -> ProjectRecord: self.log.info("Created %s record %s (%s) for %s", self.dbcli.project, prec.id, prec.name, self.who) return prec + def delete_record(self, id) -> ProjectRecord: + """ + delete the draft record. This may leave a stub record in place if, for example, the record + has been published previously. + """ + # TODO: handling previously published records + raise NotImplementedError() + def _get_id_shoulder(self, user: PubAgent): """ return an ID shoulder that is appropriate for the given user agent @@ -231,7 +239,7 @@ def update_data(self, id, newdata, part=None, message="", _prec=None): """ merge the given data into the currently save data content for the record with the given identifier. :param str id: the identifier for the record whose data should be updated. - :param str newdata: the data to save as the new content. + :param str|dict|list newdata: the data to save as the new content. :param str part: the slash-delimited pointer to an internal data property. If provided, the given ``newdata`` is a value that should be set to the property pointed to by ``part``. @@ -498,13 +506,16 @@ def _save_data(self, indata: Mapping, prec: ProjectRecord, def _validate_data(self, data): pass - def clear_data(self, id: str, part: str=None, message: str=None, prec=None): + def clear_data(self, id: str, part: str=None, message: str=None, prec=None) -> bool: """ - remove the stored data content of the record and reset it to its defaults. + remove the stored data content of the record and reset it to its defaults. Note that + no change is recorded if the requested data does not exist yet. :param str id: the identifier for the record whose data should be cleared. :param stt part: the slash-delimited pointer to an internal data property. If provided, only that property will be cleared (either removed or set to an initial default). + :return: True the data was properly cleared; return False if ``part`` was specified but does not + yet exist in the data. :param ProjectRecord prec: the previously fetched and possibly updated record corresponding to `id`. If this is not provided, the record will by fetched anew based on the `id`. :raises ObjectNotFound: if no record with the given ID exists or the `part` parameter points to @@ -518,7 +529,7 @@ def clear_data(self, id: str, part: str=None, message: str=None, prec=None): set_state = True prec = self.dbcli.get_record_for(id, ACLs.WRITE) # may raise ObjectNotFound/NotAuthorized - if _prec.status.state not in [status.EDIT, status.READY]: + if prec.status.state not in [status.EDIT, status.READY]: raise NotEditable(id) initdata = self._new_data_for(prec.id, prec.meta) @@ -541,7 +552,7 @@ def clear_data(self, id: str, part: str=None, message: str=None, prec=None): elif prop not in data: data[prop] = {} elif prop not in data: - break + return False elif not steps: del data[prop] break @@ -576,7 +587,8 @@ def clear_data(self, id: str, part: str=None, message: str=None, prec=None): finally: self._record_action(provact) self.log.info("Cleared out data for %s record %s (%s) for %s", - self.dbcli.project, _prec.id, _prec.name, self.who) + self.dbcli.project, prec.id, prec.name, self.who) + return True def update_status_message(self, id: str, message: str, _prec=None) -> status.RecordStatus: diff --git a/python/nistoar/midas/dbio/wsgi/project.py b/python/nistoar/midas/dbio/wsgi/project.py index 4489ac8..c85f099 100644 --- a/python/nistoar/midas/dbio/wsgi/project.py +++ b/python/nistoar/midas/dbio/wsgi/project.py @@ -2,13 +2,39 @@ A web service interface to various MIDAS project records. A _project record_ is a persistable record that is compliant with the MIDAS Common Database project -data model, where examples of "project record" types include DMP records and data publication drafts. +data model, where examples of a "project record" types include DMP records and data publication drafts. The :py:class:`MIDASProjectApp` encapsulates the handling of requests to create and manipulate project -records. If desired, this class can be specialized for a particular project type, and the easiest way -to do that is by sub-classing the :py:class:`~nistoar.midas.dbio.wsgi.project.ProjectRecordBroker` and -passing that class to the :py:class:`MIDASProjectApp` constructor. This is because the -:py:class:`~nistoar.midas.dbio.wsgi.project.ProjectRecordBroker` class isolates the business logic for -retrieving and manipulating project records. +records. If desired, this class can be specialized for a particular project type; as an example, see +:py:mod:`nistoar.midas.dap.service.mds3`. + +This implementation uses the simple :py:mod:`nistoar-internal WSGI +framework` to handle the specific web service endpoints. The +:py:class:`MIDASProjectApp` is the router for the Project collection endpoint: it analyzes the relative +URL path and delegates the handling to a more specific handler class. In particular, these endpoints +are handled accordingly: + +``/`` -- :py:class:`ProjectSelectionHandler` + responds to to project search queries to find project records matching search criteria (GET) + as well as accepts requests to create new records (POST). + +``/{projid}`` -- :py:class:`ProjectHandler` + returns the full project record (GET) or deletes it (DELETE). + +``/{projid}/name`` -- :py:class:`ProjectNameHandler` + returns (GET) or updates (PUT) the user-supplied name of the record. + +``/{projid}/data[/...]`` -- :py:class:`ProjectDataHandler` + returns (GET), updates (PUT, PATCH), or clears (DELETE) the data content of the record. This + implementation supports updating individual parts of the data object via PUT, PATCH, DELETE + based on the path relative to ``data``. Subclasses (e.g. with the + :py:mod:`DAP specialization`) may also support POST for certain + array-type properties within ``data``. + +``/{projid}/acls[/...]`` -- :py:class:`ProjectACLsHandler` + returns (GET) or updates (PUT, PATCH, POST, DELETE) access control lists for the record. + +``/{projid}/*`` -- :py:class`ProjectInfoHandler` + returns other non-editable parts of the record via GET (including the ``meta`` property). """ from logging import Logger from collections import OrderedDict @@ -86,7 +112,7 @@ def __init__(self, service: ProjectService, subapp: SubApp, wsgienv: dict, start raise ValueError("Missing ProjectRecord id") def do_OPTIONS(self, path): - return self.send_options(["GET"]) + return self.send_options(["GET", "DELETE"]) def do_GET(self, path, ashead=False): try: @@ -98,7 +124,22 @@ def do_GET(self, path, ashead=False): self._id, ashead=ashead) return self.send_json(prec.to_dict(), ashead=ashead) - + + def do_DELETE(self, path): + try: + prec = self.svc.get_record(self._id) + out = prec.to_dict() + self.svc.delete_record(self._id) + except dbio.NotAuthorized as ex: + return self.send_unauthorized() + except dbio.ObjectNotFound as ex: + return self.send_error_resp(404, "ID not found", "Record with requested identifier not found", + self._id) + except NotImplementedError as ex: + return self.send_error(501, "Not Implemented") + + return self.send_json(out, "Deleted") + class ProjectInfoHandler(ProjectRecordHandler): """ @@ -261,7 +302,7 @@ def __init__(self, service: ProjectService, subapp: SubApp, wsgienv: dict, start raise ValueError("Missing ProjectRecord id") def do_OPTIONS(self, path): - return self.send_options(["GET", "PUT", "PATCH"]) + return self.send_options(["GET", "PUT", "PATCH", "DELETE"]) def do_GET(self, path, ashead=False): """ @@ -283,6 +324,28 @@ def do_GET(self, path, ashead=False): "Record with requested identifier not found", self._id, ashead=ashead) return self.send_json(out, ashead=ashead) + def do_DELETE(self, path): + """ + respond to a DELETE request. This is used to clear the value of a particular property + within the project data or otherwise reset the project data to its initial defaults. + :param str path: a path to the portion of the data to clear + """ + try: + cleared = self.svc.clear_data(self._id, path) + except dbio.NotAuthorized as ex: + return self._send_unauthorized() + except dbio.PartNotAccessible as ex: + return self.send_error_resp(405, "Data part not deletable", + "Requested part of data cannot be deleted") + except dbio.ObjectNotFound as ex: + if ex.record_part: + return self.send_error_resp(404, "Data property not found", + "No data found at requested property", self._id, ashead=ashead) + return self.send_error_resp(404, "ID not found", + "Record with requested identifier not found", self._id, ashead=ashead) + + return self.send_json(cleared, "Cleared", 201) + def do_PUT(self, path): try: newdata = self.get_json_body() @@ -650,7 +713,7 @@ def do_DELETE(self, path): try: prec.acls.revoke_perm_from(parts[0], parts[1]) prec.save() - return self.send_ok() + return self.send_ok(message="ID removed") except dbio.NotAuthorized as ex: return self.send_unauthorized() @@ -787,7 +850,7 @@ def _apply_action(self, action, message=None): class MIDASProjectApp(SubApp): """ - a base web app for an interface handling project record + a base web app for an interface handling project record. """ _selection_handler = ProjectSelectionHandler _update_handler = ProjectHandler diff --git a/python/tests/nistoar/midas/dap/service/test_mds3.py b/python/tests/nistoar/midas/dap/service/test_mds3.py index affb56b..eb816fa 100644 --- a/python/tests/nistoar/midas/dap/service/test_mds3.py +++ b/python/tests/nistoar/midas/dap/service/test_mds3.py @@ -813,6 +813,45 @@ def test_get_sw_desc_for(self): "accessURL": "https://bitbucket.com/foo/bar" }) + def test_clear_data(self): + self.create_service() + prec = self.svc.create_record("goob") + pdrid = "ark:/88434/%s-%s" % tuple(prec.id.split(":")) + nerd = self.svc.get_nerdm_data(prec.id) + self.assertEqual(set(nerd.keys()), + {"_schema", "@id", "doi", "_extensionSchemas", "@context", "@type"}) + + nerd = self.svc.update_data(prec.id, + {"landingPage": "https://example.com", + "contactPoint": { "fn": "Gurn Cranston", "hasEmail": "mailto:gjc1@nist.gov"}}) + self.assertEqual(set(nerd.keys()), + {"_schema", "@id", "doi", "_extensionSchemas", "@context", "@type", + "contactPoint", "landingPage"}) + self.assertEqual(set(nerd["contactPoint"].keys()), {"@type", "fn", "hasEmail"}) + + with self.assertRaises(PartNotAccessible): + self.assertIs(self.svc.clear_data(prec.id, "goober"), False) + with self.assertRaises(PartNotAccessible): + self.assertIs(self.svc.clear_data(prec.id, "contactPoint/hasEmail"), True) + nerd = self.svc.get_nerdm_data(prec.id) + self.assertEqual(set(nerd.keys()), + {"_schema", "@id", "doi", "_extensionSchemas", "@context", "@type", + "contactPoint", "landingPage"}) + self.assertEqual(set(nerd["contactPoint"].keys()), {"@type", "fn", "hasEmail"}) + + self.assertIs(self.svc.clear_data(prec.id, "landingPage"), True) + nerd = self.svc.get_nerdm_data(prec.id) + self.assertEqual(set(nerd.keys()), + {"_schema", "@id", "doi", "_extensionSchemas", "@context", "@type", + "contactPoint"}) + self.assertIs(self.svc.clear_data(prec.id, "references"), False) + + self.assertIs(self.svc.clear_data(prec.id), True) + nerd = self.svc.get_nerdm_data(prec.id) + self.assertEqual(set(nerd.keys()), + {"_schema", "@id", "doi", "_extensionSchemas", "@context", "@type"}) + + def test_update(self): rec = read_nerd(pdr2210) self.create_service() @@ -928,6 +967,15 @@ def test_update(self): self.assertNotIn("references", nerd) self.assertEqual(len(nerd["components"]), 2) + def test_set_landingpage(self): + self.create_service() + prec = self.svc.create_record("goob") + id = prec.id + nerd = self.svc._store.open(id) + + self.svc.replace_data(id, "https://example.com/", part="landingPage") + res = nerd.get_res_data() + self.assertEqual(res.get('landingPage'), "https://example.com/") diff --git a/python/tests/nistoar/midas/dap/service/test_mds3_app.py b/python/tests/nistoar/midas/dap/service/test_mds3_app.py index bd2a3bf..1e214fb 100644 --- a/python/tests/nistoar/midas/dap/service/test_mds3_app.py +++ b/python/tests/nistoar/midas/dap/service/test_mds3_app.py @@ -383,7 +383,7 @@ def test_put_patch(self): self.assertNotIn("downloadURL", resp) self.resp = [] - path = id + '/data/components[file_1]' + path = id + '/data/components/file_1' req = { 'REQUEST_METHOD': 'GET', 'PATH_INFO': self.rootpath + path @@ -443,6 +443,93 @@ def test_put_patch(self): self.assertIn('description', resp) self.assertEqual(resp['rights'], "What ever.") + def test_delete(self): + testnerd = read_nerd(pdr2210) + res = deepcopy(testnerd) + del res['references'] + del res['components'] + del res['@id'] + del res['_schema'] + del res['_extensionSchemas'] + + path = "" + req = { + 'REQUEST_METHOD': 'POST', + 'PATH_INFO': self.rootpath + path + } + req['wsgi.input'] = StringIO(json.dumps({"data": { "contactPoint": res['contactPoint'], + "keyword": [ "testing" ], + "landingPage": "https://example.com/" }, + "meta": { "creatorisContact": "false" }, + "name": "OptSortSph" })) + hdlr = self.app.create_handler(req, self.start, path, nistr) + self.assertTrue(isinstance(hdlr, prj.ProjectSelectionHandler)) + self.assertNotEqual(hdlr.cfg, {}) + self.assertEqual(hdlr._path, "") + body = hdlr.handle() + + self.assertIn("201 ", self.resp[0]) + resp = self.body2dict(body) + self.assertEqual(resp['name'], "OptSortSph") + self.assertEqual(resp['id'], "mds3:0001") + self.assertEqual(resp['data']['@id'], 'ark:/88434/mds3-0001') + self.assertEqual(resp['data']['doi'], 'doi:10.88888/mds3-0001') + recid = resp['id'] + + self.resp = [] + path = recid + '/data' + req = { + 'REQUEST_METHOD': 'GET', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + self.assertTrue(isinstance(hdlr, prj.ProjectDataHandler)) + self.assertEqual(hdlr._path, "") + body = hdlr.handle() + self.assertIn("200 ", self.resp[0]) + resp = self.body2dict(body) + self.assertEqual(resp['@id'], 'ark:/88434/mds3-0001') + self.assertEqual(resp['doi'], 'doi:10.88888/mds3-0001') + self.assertEqual(resp['contactPoint'], + {"fn": "Zachary Levine", "@type": "vcard:Contact", + "hasEmail": "mailto:zachary.levine@nist.gov" }) + self.assertEqual(resp['@type'], + [ "nrdp:PublicDataResource", "dcat:Resource" ]) + self.assertEqual(resp['landingPage'], "https://example.com/") + self.assertEqual(resp['keyword'], ["testing"]) + + self.resp = [] + path = recid + '/data/landingPage' + req = { + 'REQUEST_METHOD': 'DELETE', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + self.assertTrue(isinstance(hdlr, prj.ProjectDataHandler)) + self.assertEqual(hdlr._path, "landingPage") + body = hdlr.handle() + self.assertIn("201 ", self.resp[0]) + resp = self.body2dict(body) + self.assertIs(resp, True) + + + self.resp = [] + path = recid + '/data' + req = { + 'REQUEST_METHOD': 'GET', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + self.assertTrue(isinstance(hdlr, prj.ProjectDataHandler)) + self.assertEqual(hdlr._path, "") + body = hdlr.handle() + self.assertIn("200 ", self.resp[0]) + resp = self.body2dict(body) + self.assertNotIn("landingPage", resp) + + + + diff --git a/python/tests/nistoar/midas/dbio/test_project.py b/python/tests/nistoar/midas/dbio/test_project.py index 74d7455..35ea008 100644 --- a/python/tests/nistoar/midas/dbio/test_project.py +++ b/python/tests/nistoar/midas/dbio/test_project.py @@ -230,6 +230,32 @@ def test_update_replace_data(self): self.assertEqual(len(self.project.dbcli._db.get(base.PROV_ACT_LOG, {}).get(prec.id,[])), 6) + def test_clear_data(self): + self.create_service() + prec = self.project.create_record("goob") + self.assertEqual(prec.data, {}) + + data = self.project.update_data(prec.id, {"color": "red", "pos": {"x": 23, "y": 12, "grid": "A"}}) + self.assertEqual(data, {"color": "red", "pos": {"x": 23, "y": 12, "grid": "A"}}) + prec = self.project.get_record(prec.id) + self.assertEqual(prec.data, {"color": "red", "pos": {"x": 23, "y": 12, "grid": "A"}}) + + self.assertIs(self.project.clear_data(prec.id, "color"), True) + prec = self.project.get_record(prec.id) + self.assertEqual(prec.data, {"pos": {"x": 23, "y": 12, "grid": "A"}}) + + self.assertIs(self.project.clear_data(prec.id, "color"), False) + self.assertIs(self.project.clear_data(prec.id, "gurn/goob/gomer"), False) + + self.assertIs(self.project.clear_data(prec.id, "pos/y"), True) + prec = self.project.get_record(prec.id) + self.assertEqual(prec.data, {"pos": {"x": 23, "grid": "A"}}) + + self.assertIs(self.project.clear_data(prec.id), True) + prec = self.project.get_record(prec.id) + self.assertEqual(prec.data, {}) + + def test_finalize(self): self.create_service() prec = self.project.create_record("goob") diff --git a/python/tests/nistoar/midas/dbio/wsgi/test_project.py b/python/tests/nistoar/midas/dbio/wsgi/test_project.py index 221c314..69e5d00 100644 --- a/python/tests/nistoar/midas/dbio/wsgi/test_project.py +++ b/python/tests/nistoar/midas/dbio/wsgi/test_project.py @@ -266,15 +266,16 @@ def test_full_methnotallowed(self): body = hdlr.handle() self.assertIn("405 ", self.resp[0]) - self.resp = [] - path = "mdm1:0001" - req = { - 'REQUEST_METHOD': 'DELETE', - 'PATH_INFO': self.rootpath + path - } - hdlr = self.app.create_handler(req, self.start, path, nistr) - body = hdlr.handle() - self.assertIn("405 ", self.resp[0]) +# DELETE is now allowed +# self.resp = [] +# path = "mdm1:0001" +# req = { +# 'REQUEST_METHOD': 'DELETE', +# 'PATH_INFO': self.rootpath + path +# } +# hdlr = self.app.create_handler(req, self.start, path, nistr) +# body = hdlr.handle() +# self.assertIn("405 ", self.resp[0]) def test_create(self): path = "" @@ -305,6 +306,86 @@ def test_create(self): self.assertEqual(resp['data'], {"color": "red"}) self.assertEqual(resp['meta'], {}) + def test_delete(self): + path = "" + req = { + 'REQUEST_METHOD': 'POST', + 'PATH_INFO': self.rootpath + path + } + req['wsgi.input'] = StringIO(json.dumps({"name": "big", "owner": "nobody", + "data": {"color": "red", "pos": {"x": 0, "y": 1}}})) + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("201 ", self.resp[0]) + resp = self.body2dict(body) + self.assertEqual(resp['data'], {"color": "red", "pos": {"x": 0, "y": 1}}) + recid = resp['id'] + + self.resp = [] + path = recid+"/data/pos/x" + req = { + 'REQUEST_METHOD': 'DELETE', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("201 ", self.resp[0]) + resp = self.body2dict(body) + self.assertIs(resp, True) + + self.resp = [] + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("201 ", self.resp[0]) + resp = self.body2dict(body) + self.assertIs(resp, False) + + self.resp = [] + path = recid+"/data" + req = { + 'REQUEST_METHOD': 'GET', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("200 ", self.resp[0]) + resp = self.body2dict(body) + self.assertEqual(resp, {"color": "red", "pos": {"y": 1}}) + + self.resp = [] + path = recid+"/data" + req = { + 'REQUEST_METHOD': 'DELETE', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("201 ", self.resp[0]) + resp = self.body2dict(body) + self.assertIs(resp, True) + + self.resp = [] + path = recid+"/data" + req = { + 'REQUEST_METHOD': 'GET', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("200 ", self.resp[0]) + resp = self.body2dict(body) + self.assertEqual(resp, {}) + + self.resp = [] + path = recid + req = { + 'REQUEST_METHOD': 'DELETE', + 'PATH_INFO': self.rootpath + path + } + hdlr = self.app.create_handler(req, self.start, path, nistr) + body = hdlr.handle() + self.assertIn("501 ", self.resp[0]) + def test_search(self): path = "" diff --git a/scripts/install.sh b/scripts/install.sh index f69e9e7..91a8f74 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -29,6 +29,7 @@ echo Installing python libraries into $PY_LIBDIR... # None at this time $oarmd_pkg/scripts/install_extras.sh --install-dir=$INSTALL_DIR +mkdir -p $INSTALL_DIR/etc/midas/schemas mkdir -p $INSTALL_DIR/var/logs echo cp -r $SOURCE_DIR/etc $INSTALL_DIR diff --git a/scripts/loosen_nerdm.py b/scripts/loosen_nerdm.py new file mode 100755 index 0000000..85f98b9 --- /dev/null +++ b/scripts/loosen_nerdm.py @@ -0,0 +1,168 @@ +#! /usr/bin/env python3 +# +import os, sys, json, argparse, traceback, re +from pathlib import Path +from collections import OrderedDict +from collections.abc import Mapping + +description="""create copies of NERDm schemas with loosened requirements appropriate for the +metadata drafting process (i.e. within MIDAS)""" +epilog="" +def_progname = "loosen_nerdm" + +def define_options(progname, parser=None): + """ + define command-line arguments + """ + if not parser: + parser = argparse.ArgumentParser(progname, None, description, epilog) + + parser.add_argument("srcdir", metavar="SRCDIR", type=str, + help="the directory containing the NERDm schemas") + parser.add_argument("destdir", metavar="DESTDIR", type=str, + help="the directory write loosened schemas to") + parser.add_argument("-D", "--no-dedocument", dest="dedoc", action="store_false", default=True, + help="do not remove documentation from source schemas") + parser.add_argument("-J", "--assume-post2020", dest="post2020", action="store_true", default=False, + help="assume schemas are compliant with a post-2020 JSON Schema specification "+ + "(and uses $defs)") + parser.add_argument("-m", "--make-dest-dir", dest="mkdest", action="store_true", default=False, + help="create the destination directory if it does not exist") + + return parser + +def set_options(progname, args): + """ + define and parse the command-line options + """ + return define_options(progname).parse_args(args) + +directives_by_file = { + "nerdm-schema.json": { + "derequire": [ "Resource", "Organization" ] + }, + "nerdm-pub-schema.json": { + "derequire": [ "PublicDataResource", "Person" ] + }, + "nerdm-rls-schema.json": { + "derequire": [ "ReleasedResource" ] + } +} + +def find_nistoar_code(): + execdir = Path(__file__).resolve().parents[0] + basedir = execdir.parents[0] + mdpydir = basedir / "metadata" / "python" + return mdpydir + +try: + import nistoar.nerdm.utils as nerdm_utils +except ImportError: + sys.path.insert(0, str(find_nistoar_code())) + import nistoar.nerdm.utils as nerdm_utils + +def loosen_schema(schema: Mapping, directives: Mapping, opts=None): + """ + apply the given loosening directive to the given JSON Schema. The directives is a + dictionary describes what to do with the following properties (the directives) supported: + + ``derequire`` + a list of type definitions within the schema from which the required property + should be removed (via :py:func:`~nistoar.nerdm.utils.unrequire_props_in`). Each + type name listed will be assumed to be an item under the "definitions" node in the + schema this directive is applied to. + ``dedocument`` + a boolean indicating whether the documentation annotations should be removed from + the schema. If not set, the default is determined by opts.dedoc if opts is given or + True, otherwise. + + :param dict schema: the schema document as a JSON Schema schema dictionary + :param dict directives: the dictionary of directives to apply + :param opt: an options object (containing scripts command-line options) + """ + if opts: + if not opts.dedoc: + directives["dedocument"] = False + directives["post2020"] = opts.post2020 + + nerdm_utils.loosen_schema(schema, directives) + +def process_nerdm_schemas(srcdir, destdir, opts=None): + """ + process all NERDm schemas (core and extensions) found in the source directory + and write the modified schemas to the output directory + """ + if not os.path.isdir(srcdir): + raise RuntimeException(f"{srcdir}: schema source directory does not exist as directory") + + if not os.path.exists(destdir): + if opts and opts.mkdest: + os.makedirs(destdir) + else: + raise FileNotFoundError(destdir) + if not os.path.isdir(srcdir): + raise RuntimeException(f"{destdir}: schema destination is not a directory") + + nerdfilere = re.compile(r"^nerdm-([a-zA-Z][^\-]*\-)?schema.json$") + schfiles = [f for f in os.listdir(srcdir) if nerdfilere.match(f)] + + failed={} + for f in schfiles: + try: + with open(os.path.join(srcdir, f)) as fd: + schema = json.load(fd, object_pairs_hook=OrderedDict) + except IOError as ex: + failed[f] = f"Trouble reading schema file: {str(ex)}" + continue + + directives = directives_by_file.get(f, {}) + try: + loosen_schema(schema, directives, opts) + except Exception as ex: + failed[f] = f"Trouble processing schema file: {str(ex)}" + continue + + with open(os.path.join(destdir, f), 'w') as fd: + json.dump(schema, fd, indent=2) + fd.write("\n") + + return failed + +def main(progname=None, args=[]): + global def_progname; + if not progname: + progname = def_progname + else: + progname = os.path.basename(progname) + if progname.endswith(".py"): + progname = progname[:-1*len(".py")] + + opts = set_options(progname, args) + + failed = process_nerdm_schemas(opts.srcdir, opts.destdir, opts) # may raise exceptions + if failed: + print(f"{progname}: WARNING: failed to process the following schemas:", file=sys.stderr) + for f, err in failed: + print(f" {f}: {err}", file=sys.stderr) + + return 3 + + return 0 + +if __name__ == "__main__": + try: + sys.exit(main(sys.argv[0], sys.argv[1:])) + except RuntimeError as ex: + print(f"{progname}: {str(ex)}", file=sys.stderr) + sys.exit(1) + except Exception as ex: + print("Unexpected error: "+str(ex), file=sys.stderr) + traceback.print_tb(sys.exc_info()[2]) + sys.exit(4) + + + + + + + diff --git a/scripts/tests/test_loosen_nerdm.py b/scripts/tests/test_loosen_nerdm.py new file mode 100644 index 0000000..2ba8f29 --- /dev/null +++ b/scripts/tests/test_loosen_nerdm.py @@ -0,0 +1,185 @@ +#! /usr/bin/env python3 +# +import sys, os, csv, json, re +import importlib.util as imputil +import unittest as test +from pathlib import Path +from collections import OrderedDict + +from nistoar.testing import * +from nistoar.base.config import hget + +testdir = Path(__file__).resolve().parents[0] +scrpdir = testdir.parents[0] +basedir = scrpdir.parents[0] +nerdmdir = basedir / "metadata" / "model" + +scriptfile = str(scrpdir / "loosen_nerdm.py") + +def import_file(path, name=None): + if not name: + name = os.path.splitext(os.path.basename(path))[0] + import importlib.util as imputil + spec = imputil.spec_from_file_location(name, path) + out = imputil.module_from_spec(spec) + sys.modules["loosen"] = out + spec.loader.exec_module(out) + return out + +loosen = None # set at end of this file + +def setUpModule(): + ensure_tmpdir() + +def tearDownModule(): + rmtmpdir() + + +class TestLoosenNerdm(test.TestCase): + + def test_import(self): + self.assertIsNotNone(loosen) + self.assertTrue(hasattr(loosen, 'main')) + self.assertIsNotNone(loosen.directives_by_file) + + def setUp(self): + self.tf = Tempfiles() + self.destdir = self.tf.mkdir("loosen_nerdm") + + def tearDown(self): + self.tf.clean() + + def test_set_options(self): + try: + opts = loosen.set_options(loosen.def_progname, ["-D", "goob", "gurn"]) + self.assertFalse(opts.dedoc) + self.assertFalse(opts.post2020) + self.assertEqual(opts.srcdir, "goob") + self.assertEqual(opts.destdir, "gurn") + + opts = loosen.set_options(loosen.def_progname, ["-J", "goob", "gurn"]) + self.assertTrue(opts.dedoc) + self.assertTrue(opts.post2020) + self.assertEqual(opts.srcdir, "goob") + self.assertEqual(opts.destdir, "gurn") + + opts = loosen.set_options(loosen.def_progname, ["harry", "david"]) + self.assertTrue(opts.dedoc) + self.assertFalse(opts.post2020) + self.assertEqual(opts.srcdir, "harry") + self.assertEqual(opts.destdir, "david") + except SystemExit as ex: + self.fail("error processing args") + + def test_find_nistoar_code(self): + self.assertEqual(loosen.find_nistoar_code().parts[-2:], ("metadata", "python")) + + def test_loosen_schema(self): + with open(nerdmdir/"nerdm-schema.json") as fd: + schema = json.load(fd, object_pairs_hook=OrderedDict) + + self.assertTrue(hget(schema, "title")) + self.assertTrue(hget(schema, "description")) + self.assertTrue(hget(schema, "definitions.Resource.required")) + self.assertTrue(hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(hget(schema, "definitions.Organization.description")) + + loosen.loosen_schema(schema, {"derequire": ["Resource"], "dedocument": True}) + + self.assertTrue(not hget(schema, "title")) + self.assertTrue(not hget(schema, "description")) + self.assertTrue(not hget(schema, "definitions.Resource.required")) + self.assertTrue(not hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(not hget(schema, "definitions.Organization.description")) + + def test_loosen_schema_no_dedoc(self): + with open(nerdmdir/"nerdm-schema.json") as fd: + schema = json.load(fd, object_pairs_hook=OrderedDict) + + self.assertTrue(hget(schema, "title")) + self.assertTrue(hget(schema, "description")) + self.assertTrue(hget(schema, "definitions.Resource.required")) + self.assertTrue(hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(hget(schema, "definitions.Organization.description")) + + loosen.loosen_schema(schema, {"derequire": ["Resource"], "dedocument": False}) + + self.assertTrue(hget(schema, "title")) + self.assertTrue(hget(schema, "description")) + self.assertTrue(not hget(schema, "definitions.Resource.required")) + self.assertTrue(hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(hget(schema, "definitions.Organization.description")) + + def test_loosen_schema_with_opts(self): + with open(nerdmdir/"nerdm-schema.json") as fd: + schema = json.load(fd, object_pairs_hook=OrderedDict) + opts = loosen.set_options(loosen.def_progname, ["goob", "gurn"]) + + self.assertTrue(hget(schema, "title")) + self.assertTrue(hget(schema, "description")) + self.assertTrue(hget(schema, "definitions.Resource.required")) + self.assertTrue(hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(hget(schema, "definitions.Organization.description")) + + loosen.loosen_schema(schema, {"derequire": ["Resource"]}, opts) + + self.assertTrue(not hget(schema, "title")) + self.assertTrue(not hget(schema, "description")) + self.assertTrue(not hget(schema, "definitions.Resource.required")) + self.assertTrue(not hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(not hget(schema, "definitions.Organization.description")) + + def test_loosen_schema_with_opts_D(self): + with open(nerdmdir/"nerdm-schema.json") as fd: + schema = json.load(fd, object_pairs_hook=OrderedDict) + opts = loosen.set_options(loosen.def_progname, ["-D", "goob", "gurn"]) + + self.assertTrue(hget(schema, "title")) + self.assertTrue(hget(schema, "description")) + self.assertTrue(hget(schema, "definitions.Resource.required")) + self.assertTrue(hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(hget(schema, "definitions.Organization.description")) + + loosen.loosen_schema(schema, {"derequire": ["Resource"]}, opts) + + self.assertTrue(hget(schema, "title")) + self.assertTrue(hget(schema, "description")) + self.assertTrue(not hget(schema, "definitions.Resource.required")) + self.assertTrue(hget(schema, "definitions.Resource.description")) + self.assertTrue(hget(schema, "definitions.Organization.required")) + self.assertTrue(hget(schema, "definitions.Organization.description")) + + def test_process_nerdm_schemas(self): + schfre = re.compile(r"^nerdm-([a-zA-Z][^\-]*\-)?schema.json$") + srcfiles = [f for f in os.listdir(nerdmdir) if schfre.match(f)] + self.assertGreater(len(srcfiles), 6) + + destfiles = [f for f in os.listdir(self.destdir) if not f.startswith('.')] + self.assertEqual(destfiles, []) + self.assertEqual(loosen.process_nerdm_schemas(nerdmdir, self.destdir), {}) + + destfiles = [f for f in os.listdir(self.destdir) if not f.startswith('.')] + self.assertIn("nerdm-schema.json", destfiles) + self.assertIn("nerdm-pub-schema.json", destfiles) + for schfile in srcfiles: + self.assertIn(schfile, destfiles) + self.assertEqual(len(destfiles), len(srcfiles)) + + + + + + + +if __name__ == '__main__': + if len(sys.argv) > 1: + scriptfile = sys.argv[1] + loosen = import_file(scriptfile) + test.main()