diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a8423d665a0e549247e2805785de5953dd712b62..ca7d3e3d8eb2ba2ad07f1a74314414a41174f334 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,9 +85,10 @@ python tests: alias: rabbitmq - name: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 alias: elastic - # fix issue with running elastic in gitlab ci runner: - # https://gitlab.com/gitlab-org/gitlab-ce/issues/42214 - command: [ "bin/elasticsearch", "-Ediscovery.type=single-node" ] + command: + - bash + - "-c" + - ES_JAVA_OPTS="-Xms512m -Xmx512m" docker-entrypoint.sh elasticsearch -Ediscovery.type=single-node -Expack.security.enabled=false - name: mongo:5.0.6 alias: mongo variables: @@ -101,6 +102,8 @@ python tests: NOMAD_KEYCLOAK_PASSWORD: ${CI_KEYCLOAK_ADMIN_PASSWORD} NOMAD_NORMALIZE_SPRINGER_DB_PATH: /nomad/fairdi/db/data/springer.msg script: + - sleep 5 + - curl http://elastic:9200/_cat/health - cd /app - ls /builds - python -m pytest --cov=nomad -sv tests diff --git a/dependencies/parsers/atomistic b/dependencies/parsers/atomistic index 958d4f2ef91563791ebfe6b329fae5615d992024..7a57120d6b4845f471577af75ad3e205ee503ea5 160000 --- a/dependencies/parsers/atomistic +++ b/dependencies/parsers/atomistic @@ -1 +1 @@ -Subproject commit 958d4f2ef91563791ebfe6b329fae5615d992024 +Subproject commit 7a57120d6b4845f471577af75ad3e205ee503ea5 diff --git a/dependencies/parsers/electronic b/dependencies/parsers/electronic index 122d8347e79cd4e77adfcc6db4962b10b6075801..eee9bd051c3c52e3560cdb8092f79488dee05009 160000 --- a/dependencies/parsers/electronic +++ b/dependencies/parsers/electronic @@ -1 +1 @@ -Subproject commit 122d8347e79cd4e77adfcc6db4962b10b6075801 +Subproject commit eee9bd051c3c52e3560cdb8092f79488dee05009 diff --git a/dependencies/parsers/workflow b/dependencies/parsers/workflow index d9d499b873b1bd1847a13376acc48a928661036a..4050431cce67d80793e548e7f79e44de7e95ecb8 160000 --- a/dependencies/parsers/workflow +++ b/dependencies/parsers/workflow @@ -1 +1 @@ -Subproject commit d9d499b873b1bd1847a13376acc48a928661036a +Subproject commit 4050431cce67d80793e548e7f79e44de7e95ecb8 diff --git a/docs/oasis.md b/docs/oasis.md index d1879f2999fd9c8ceb7dea737a59a48271fcc8d3..e358659fd0a183d8a3b2ad5011e696a8c9aa27f1 100644 --- a/docs/oasis.md +++ b/docs/oasis.md @@ -103,6 +103,7 @@ services: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 container_name: nomad_oasis_elastic environment: + - ES_JAVA_OPTS=-Xms512m -Xmx512m - discovery.type=single-node volumes: - nomad_oasis_elastic:/usr/share/elasticsearch/data diff --git a/gui/package.json b/gui/package.json index 9c8c5baac3f0e22877a260e9b00ed2310ad90bcb..1f5282998e562b62ddf2d51cf4f419c1f9147e75 100644 --- a/gui/package.json +++ b/gui/package.json @@ -1,6 +1,6 @@ { "name": "nomad-fair-gui", - "version": "1.0.6", + "version": "1.0.7", "commit": "e98694e", "private": true, "workspaces": [ diff --git a/gui/public/env.js b/gui/public/env.js index 673273ffba004db2a4c8d30cbb6254b011522120..e522694959bb1c8c8be0bbc2dcb788489ecbdf16 100644 --- a/gui/public/env.js +++ b/gui/public/env.js @@ -11,7 +11,7 @@ window.nomadEnv = { 'encyclopediaBase': 'https://nomad-lab.eu/prod/rae/encyclopedia/#', 'debug': false, 'version': { - 'label': '1.0.6', + 'label': '1.0.7', 'isBeta': false, 'isTest': true, 'usesBetaData': true, diff --git a/gui/src/setupTests.js b/gui/src/setupTests.js index b169ea0c7222c95b79c02a02e7762adea00482c2..03ed708b22ae79f9acc96ffb2b352f83b7ddb08c 100644 --- a/gui/src/setupTests.js +++ b/gui/src/setupTests.js @@ -11,7 +11,7 @@ global.nomadEnv = { 'appBase': 'http://nomad-lab.eu/prod/rae/beta', 'debug': false, 'version': { - 'label': '1.0.6', + 'label': '1.0.7', 'isBeta': false, 'isTest': true, 'usesBetaData': true, diff --git a/nomad/app/main.py b/nomad/app/main.py index cc46d85c48e64497f658358dbdbe706c55892931..eadea483f13c89439a34d4445fd7d2241856195c 100644 --- a/nomad/app/main.py +++ b/nomad/app/main.py @@ -68,6 +68,9 @@ app.mount(app_base, WSGIMiddleware(flask_app)) @app.on_event('startup') async def startup_event(): + from nomad.parsing.parsers import import_all_parsers + import_all_parsers() + from nomad import infrastructure # each subprocess is supposed disconnect connect again: https://jira.mongodb.org/browse/PYTHON-2090 try: diff --git a/nomad/cli/dev.py b/nomad/cli/dev.py index d83564c1726aa8253499037f5174ba6654d337d9..db2c03735c4fdc0806acb2129f8c479ccbb893a1 100644 --- a/nomad/cli/dev.py +++ b/nomad/cli/dev.py @@ -77,8 +77,8 @@ def _all_metainfo_packages(): # TODO similar to before, due to lazyloading, we need to explicily access parsers # to actually import all parsers and indirectly all metainfo packages - from nomad.parsing import parsers - parsers.parsers + from nomad.parsing.parsers import import_all_parsers + import_all_parsers() # Create the ES mapping to populate ES annoations with search keys. from nomad.search import entry_type diff --git a/nomad/config.py b/nomad/config.py index 892549524c812ef202eb417191e2615994e0c8cb..f11340314debf658b4e06ff6f62fd87c50c5bb0d 100644 --- a/nomad/config.py +++ b/nomad/config.py @@ -311,7 +311,7 @@ datacite = NomadConfig( ) meta = NomadConfig( - version='1.0.6', + version='1.0.7', commit=gitinfo.commit, deployment='devel', label=None, diff --git a/nomad/datamodel/datamodel.py b/nomad/datamodel/datamodel.py index 093073c8d65f827340b7745d8e83ff111197b9d7..d2945239cc7bb231442560408b1b9e7f8add947a 100644 --- a/nomad/datamodel/datamodel.py +++ b/nomad/datamodel/datamodel.py @@ -551,7 +551,7 @@ class EntryMetadata(metainfo.MSection): a_elasticsearch=Elasticsearch()) external_db = metainfo.Quantity( - type=metainfo.MEnum('EELS Data Base', 'Materials Project', 'AFLOW', 'OQMD'), + type=metainfo.MEnum('EELS Data Base', 'Materials Project', 'AFLOW', 'OQMD', 'Kyoto Phonopy Database'), categories=[MongoUploadMetadata, EditableUserMetadata], description='The repository or external database where the original data resides', a_elasticsearch=Elasticsearch(material_entry_type)) diff --git a/nomad/datamodel/metainfo/simulation/calculation.py b/nomad/datamodel/metainfo/simulation/calculation.py index dad071e6aaea6ee7b494a5e5bedb736990c216cc..0198b734ec5240abe1959467e32b7dfcccbe98a3 100644 --- a/nomad/datamodel/metainfo/simulation/calculation.py +++ b/nomad/datamodel/metainfo/simulation/calculation.py @@ -310,7 +310,7 @@ class Energy(MSection): energy calculated with the functional stored in XC_functional. ''') - # TODO Could potential be generalized for other energy types + # TODO Remove this should use xc.potential xc_potential = SubSection( sub_section=EnergyEntry.m_def, description=''' @@ -335,6 +335,7 @@ class Energy(MSection): energy. ''') + # TODO remove this or electrostatic coulomb = SubSection( sub_section=EnergyEntry.m_def, description=''' @@ -528,6 +529,38 @@ class Forces(MSection): corresponding to the minus gradient of energy_T0. ''') + enthalpy = Quantity( + type=np.dtype(np.float64), + shape=[], + unit='joule', + description=''' + Value of the calculated enthalpy per cell i.e. energy_total + pressure * volume. + ''') + + entropy = Quantity( + type=np.dtype(np.float64), + shape=[], + unit='joule / kelvin', + description=''' + Value of the entropy. + ''') + + chemical_potential = Quantity( + type=np.dtype(np.float64), + shape=[], + unit='joule', + description=''' + Value of the chemical potential. + ''') + + internal = Quantity( + type=np.dtype(np.float64), + shape=[], + unit='joule', + description=''' + Value of the internal energy. + ''') + contributions = SubSection( sub_section=ForcesEntry.m_def, description=''' @@ -1129,6 +1162,7 @@ class Multipoles(MSection): higher_order = SubSection(sub_section=MultipolesEntry.m_def, repeats=True) +# TODO remove this section class Thermodynamics(MSection): ''' Section containing results related to a thermodynamics calculation. @@ -1564,6 +1598,37 @@ class BaseCalculation(MSection): contamination in spin-unrestricted calculations. ''') + pressure = Quantity( + type=np.dtype(np.float64), + shape=[], + unit='pascal', + description=''' + Value of the pressure of the system. + ''') + + temperature = Quantity( + type=np.dtype(np.float64), + shape=[], + unit='kelvin', + description=''' + Value of the temperature of the system at which the properties are calculated. + ''') + + time_physical = Quantity( + type=np.dtype(np.int32), + shape=[], + unit='second', + description=''' + The elapsed time with respect to the start of the simulation. + ''') + + time_step = Quantity( + type=np.dtype(np.int32), + shape=[], + description=''' + The number of time steps with respect to the start of the simulation. + ''') + energy = SubSection(sub_section=Energy.m_def, categories=[FastAccess]) forces = SubSection(sub_section=Forces.m_def) diff --git a/nomad/datamodel/metainfo/simulation/method.py b/nomad/datamodel/metainfo/simulation/method.py index e33e1e0acd047476feb3ae6e06e3da4c93b9c62b..7df161153dd15db03a26f97e8fe5a1f1c4e56f4b 100644 --- a/nomad/datamodel/metainfo/simulation/method.py +++ b/nomad/datamodel/metainfo/simulation/method.py @@ -232,7 +232,7 @@ class MoleculeParameters(MSection): Number of atoms in the molecule. ''') - atoms = SubSection(sub_section=AtomParameters.m_def, repeats=True) + atom_parameters = SubSection(sub_section=AtomParameters.m_def, repeats=True) class GaussianBasisGroup(MSection): @@ -432,8 +432,7 @@ class Interaction(MSection): type=str, shape=[], description=''' - Denotes the classification of the potential. Could be one of pair, LJ, Morse, EAM, - MEAM. + Denotes the classification of the potential. ''') name = Quantity( @@ -444,16 +443,24 @@ class Interaction(MSection): cut-offs, potential versions. ''') + n_atoms = Quantity( + type=np.dtype(np.int32), + shape=[], + description=''' + Number of atoms included in the interaction + ''' + ) + atom_labels = Quantity( type=str, - shape=[2], + shape=['n_atoms'], description=''' Labels of the atoms described by the interaction. ''') atom_indices = Quantity( type=np.dtype(np.int32), - shape=[2], + shape=['n_atoms'], description=''' Indices of the atoms in the system described by the interaction. ''') diff --git a/nomad/datamodel/metainfo/simulation/system.py b/nomad/datamodel/metainfo/simulation/system.py index 4fe2a44ad2185cc3f6586c555cd690934c30c6b4..b42f30d8193309a7f93b28ced703e4ab5e9e03ba 100644 --- a/nomad/datamodel/metainfo/simulation/system.py +++ b/nomad/datamodel/metainfo/simulation/system.py @@ -399,7 +399,7 @@ class Constraint(MSection): Number of atoms involved in this constraint. ''') - indices = Quantity( + atom_indices = Quantity( type=np.dtype(np.int32), shape=['n_constraints', 'n_atoms'], description=''' diff --git a/nomad/infrastructure.py b/nomad/infrastructure.py index 3b88cd5e58b50df6037976357ef948a247e1eeb0..fb65ac27e9c2b30fa1df0e7d2e3b808b6c549cc5 100644 --- a/nomad/infrastructure.py +++ b/nomad/infrastructure.py @@ -308,6 +308,7 @@ class Keycloak(): kwargs = {key: value[0] for key, value in keycloak_user.get('attributes', {}).items()} oasis_admin = kwargs.pop('is_oasis_admin', None) is not None return datamodel.User( + m_ignore_additional_keys=True, user_id=keycloak_user['id'], email=keycloak_user.get('email'), username=keycloak_user.get('username'), diff --git a/nomad/metainfo/metainfo.py b/nomad/metainfo/metainfo.py index 5e15be5f7df46732062aef132b3062abcc57f3f1..4279e9e861fff423c5bbe7b48b7115c90f80320e 100644 --- a/nomad/metainfo/metainfo.py +++ b/nomad/metainfo/metainfo.py @@ -1292,30 +1292,28 @@ class MSection(metaclass=MObjectMeta): # TODO find a way to make this a subclas return cast(MSectionBound, sub_section) - def m_update(self, safe: bool = True, **kwargs): + def m_update(self, m_ignore_additional_keys: bool = False, **kwargs): ''' Updates all quantities and sub-sections with the given arguments. ''' self.m_mod_count += 1 - if safe: - for name, value in kwargs.items(): - prop = self.m_def.all_aliases.get(name, None) - if prop is None: - raise KeyError('%s is not an attribute of this section %s' % (name, self)) - - if isinstance(prop, SubSection): - if prop.repeats: - if isinstance(value, List): - for item in value: - self.m_add_sub_section(prop, item) - else: - raise TypeError('Sub section %s repeats, but no list was given' % prop.name) - else: - self.m_add_sub_section(prop, value) + for name, value in kwargs.items(): + prop = self.m_def.all_aliases.get(name, None) + if prop is None: + if m_ignore_additional_keys: + continue + raise KeyError('%s is not an attribute of this section %s' % (name, self)) + if isinstance(prop, SubSection): + if prop.repeats: + if isinstance(value, List): + for item in value: + self.m_add_sub_section(prop, item) + else: + raise TypeError('Sub section %s repeats, but no list was given' % prop.name) else: - self.m_set(prop, value) + self.m_add_sub_section(prop, value) - else: - self.__dict__.update(**kwargs) + else: + self.m_set(prop, value) def m_as(self, section_cls: Type[MSectionBound]) -> MSectionBound: ''' 'Casts' this section to the given extending sections. ''' diff --git a/nomad/parsing/parser.py b/nomad/parsing/parser.py index 0a65ae2058bd8f67eb87ef71baf16452a09200d9..30d20e7b0f1ed71442a7ae9aa055c847fbb17d9c 100644 --- a/nomad/parsing/parser.py +++ b/nomad/parsing/parser.py @@ -237,18 +237,27 @@ class MatchingParserInterface(MatchingParser): def mainfile_parser(self): if self._mainfile_parser is None: try: - module_path, parser_class = self._parser_class_name.rsplit('.', 1) - module = importlib.import_module(module_path) - self._mainfile_parser = getattr(module, parser_class)() + Parser = self.import_parser_class() + self._mainfile_parser = Parser() except Exception as e: logger = utils.get_logger(__name__) - logger.error('Error importing parser.', exc_info=e) + logger.error('cannot instantiate parser.', exc_info=e) raise e return self._mainfile_parser def parse(self, mainfile: str, archive: EntryArchive, logger=None): self.mainfile_parser.parse(mainfile, archive, logger) + def import_parser_class(self): + try: + module_path, parser_class = self._parser_class_name.rsplit('.', 1) + module = importlib.import_module(module_path) + return getattr(module, parser_class) + except Exception as e: + logger = utils.get_logger(__name__) + logger.error('cannot import parser', exc_info=e) + raise e + class ArchiveParser(MatchingParser): def __init__(self): diff --git a/nomad/parsing/parsers.py b/nomad/parsing/parsers.py index 63ecbfbb16a1c203e1c471dc8b5bab94b1f6ee34..b9a8076b1f3ec0607270b21f6e72086aec4ae836 100644 --- a/nomad/parsing/parsers.py +++ b/nomad/parsing/parsers.py @@ -545,3 +545,13 @@ for parser in parsers: code_names = sorted(set(code_names), key=lambda code_name: code_name.lower()) results.Simulation.program_name.a_elasticsearch[0].values = code_names + [ config.services.unavailable_value, config.services.not_processed_value] + + +def import_all_parsers(): + ''' + Imports all the parsers. This will instantiate all parser metainfo as a side + effect. + ''' + for parser in parsers: + if isinstance(parser, MatchingParserInterface): + parser.import_parser_class() diff --git a/ops/docker-compose/infrastructure/docker-compose.yml b/ops/docker-compose/infrastructure/docker-compose.yml index 14b49d390e97652611c21c00493896e00df89630..c97c6df384c822ba64bf6c1281d1681c690be1f9 100644 --- a/ops/docker-compose/infrastructure/docker-compose.yml +++ b/ops/docker-compose/infrastructure/docker-compose.yml @@ -48,11 +48,13 @@ services: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.1 container_name: nomad_elastic environment: + - ES_JAVA_OPTS=-Xms512m -Xmx512m - cluster.routing.allocation.disk.threshold_enabled=true - cluster.routing.allocation.disk.watermark.flood_stage=1gb - cluster.routing.allocation.disk.watermark.low=4gb - cluster.routing.allocation.disk.watermark.high=2gb - discovery.type=single-node + - xpack.security.enabled=false ports: - 9200:9200 volumes: diff --git a/ops/helm/nomad/Chart.yaml b/ops/helm/nomad/Chart.yaml index 77444b47ba874860871bbb72ce5edf9a1595aded..d9cc2e12e65b51b43e0f77e06b9429e713e4a448 100644 --- a/ops/helm/nomad/Chart.yaml +++ b/ops/helm/nomad/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "1.0.6" +appVersion: "1.0.7" description: A Helm chart for Kubernetes that only runs nomad services and uses externally hosted databases. name: nomad -version: "1.0.6" +version: "1.0.7" diff --git a/ops/helm/nomad/deployments/prod-staging-values.yaml b/ops/helm/nomad/deployments/prod-staging-values.yaml index fe744d2d23b226a8c5f86e3a2a58bac800d8107b..4831d21a33da004177ba2fa31fd71321b9eda346 100644 --- a/ops/helm/nomad/deployments/prod-staging-values.yaml +++ b/ops/helm/nomad/deployments/prod-staging-values.yaml @@ -1,5 +1,5 @@ version: - label: "1.0.6" + label: "1.0.7" isBeta: true usesBetaData: false officialUrl: "https://nomad-lab.eu/prod/rae/gui" diff --git a/ops/helm/nomad/deployments/prod-test-values.yaml b/ops/helm/nomad/deployments/prod-test-values.yaml index 83c73eaf9cc67b34df174268825e5db1a4fd1635..c82fe644a7430f770daf1df34dcba27b3970ddd1 100644 --- a/ops/helm/nomad/deployments/prod-test-values.yaml +++ b/ops/helm/nomad/deployments/prod-test-values.yaml @@ -1,5 +1,5 @@ version: - label: "1.0.6" + label: "1.0.7" isBeta: true usesBetaData: true officialUrl: "https://nomad-lab.eu/prod/rae/gui" diff --git a/ops/helm/nomad/deployments/prod-v1-values.yaml b/ops/helm/nomad/deployments/prod-v1-values.yaml index ab1cb3289eaa60e8f8fd3d4336484aa465d1a5c8..6ec2c50eca88496d658c3158adac95124da7431f 100644 --- a/ops/helm/nomad/deployments/prod-v1-values.yaml +++ b/ops/helm/nomad/deployments/prod-v1-values.yaml @@ -1,5 +1,5 @@ version: - label: "1.0.6" + label: "1.0.7" isBeta: true usesBetaData: false officialUrl: "https://nomad-lab.eu/prod/rae/gui" diff --git a/ops/helm/nomad/values.yaml b/ops/helm/nomad/values.yaml index c3b5bb00b4f6a04c00af065a15032fdcdb71017c..fdf9e90383c96706b3dc5497c7d98d758f344172 100644 --- a/ops/helm/nomad/values.yaml +++ b/ops/helm/nomad/values.yaml @@ -1,6 +1,6 @@ ## Default values for nomad@FAIRDI version: - label: "1.0.6" + label: "1.0.7" isTest: false isBeta: false usesBetaData: false