Feature/search internal remove

autofit/config/general.yaml

@@ -10,21 +10,19 @@ model:
   ignore_prior_limits: false        # If ``True`` the limits applied to priors will be ignored, where limits set upper / lower limits. This stops PriorLimitException's from being raised.
 output:
   force_pickle_overwrite: false     # If True pickle files output by a search (e.g. samples.pickle) are recreated when a new model-fit is performed.
-  force_visualize_overwrite: false # If True, visualization images output by a search (e.g. subplots of the fit) are recreated when a new model-fit is performed.
+  force_visualize_overwrite: false  # If True, visualization images output by a search (e.g. subplots of the fit) are recreated when a new model-fit is performed.
   info_whitespace_length: 80        # Length of whitespace between the parameter names and values in the model.info / result.info
   log_level: INFO                   # The level of information output by logging.
   log_to_file: false                # If True, outputs the non-linear search log to a file (and not printed to screen).
   log_file: output.log              # The name of the file the logged output is written to (in the non-linear search output folder)
   model_results_decimal_places: 3   # Number of decimal places estimated parameter values / errors are output in model.results.
   remove_files: false               # If True, all output files of a non-linear search (e.g. samples, visualization, etc.) are deleted once the model-fit has completed, such that only the .zip file remains.
+  search_internal : false    # If True, the search internal folder which contains a saved state of the non-linear search is delete, saving on hard-disk space.
   samples_to_csv: true              # If True, non-linear search samples are written to a .csv file.
   unconverged_sample_size : 100     # If outputting results of an unconverged search, the number of samples used to estimate the median PDF values and errors.
 parallel:
   warn_environment_variables: true  # If True, a warning is displayed when the search's number of CPU > 1 and enviromment variables related to threading are also > 1.
-prior_passer:
-  sigma: 3.0                        # For non-linear search chaining and model prior passing, the sigma value of the inferred model parameter used as the sigma of the passed Gaussian prior.
-  use_errors: true                  # If True, the errors of the previous model's results are used when passing priors.
-  use_widths: true                  # If True the width of the model parameters defined in the priors config file are used.
+
 profiling:
   parallel_profile: false           # If True, the parallelization of the fit is profiled outputting a cPython graph.
   should_profile: false             # If True, the ``profile_log_likelihood_function()`` function of an analysis class is called throughout a model-fit, profiling run times.

autofit/config/output.yaml

@@ -6,4 +6,42 @@
 
 # If a given file is not listed then the default value is used.
 
-default: true # If true then files which are not explicitly listed here are output anyway. If false then they are not.
+default: true # If true then files which are not explicitly listed here are output anyway. If false then they are not.
+
+### Samples ###
+
+# The `samples.csv`file contains every sampled value of every free parameter with its log likelihood and weight.
+
+# This file is often large, therefore disabling it can significantly reduce hard-disk space use.
+
+# `samples.csv` is used to perform marginalization, infer model parameter errors and do other analysis of the search
+# chains. Even if output of `samples.csv` is disabled, these tasks are still performed by the fit and output to
+# the `samples_summary.json` file. However, without a `samples.csv` file these types of tasks cannot be performed
+# after the fit is complete, for example via the database.
+
+samples: true
+
+### Search Internal ###
+
+# The search internal folder which contains a saved state of the non-linear search, as a .pickle or .dill file.
+
+# If the entry below is false, the folder is still output during the model-fit, as it is required to resume the fit
+# from where it left off. Therefore, settings `false` below does not impact model-fitting checkpointing and resumption.
+# Instead, the search internal folder is deleted once the fit is completed.
+
+# The search internal folder file is often large, therefore deleting it after a fit is complete can significantly
+# reduce hard-disk space use.
+
+# The search internal representation (e.g. what you can load from the output .pickle file) may have additional
+# quantities specific to the non-linear search that you are interested in inspecting. Deleting the folder means this
+# information is list.
+
+search_internal: true
+
+# Other Files:
+
+covariance: true # `covariance.csv`: The [free parameters x free parameters] covariance matrix.
+derived_quantities: true # `derived_quantities.csv`: Values of every derived quantity, which are not model free parameter but values computed from them.
+data: true # `data.json`: The value of every data point in the data.
+noise_map: true # `noise_map.json`: The value of every RMS noise map value.
+

autofit/example/analysis.py

@@ -1,5 +1,3 @@
-import json
-from os import path
 import os
 import matplotlib.pyplot as plt
 from typing import List

autofit/mapper/prior_model/abstract.py

@@ -905,8 +905,8 @@ def mapper_from_prior_arguments(self, arguments):
     def gaussian_prior_model_for_arguments(self, arguments):
         raise NotImplementedError()
 
-    def mapper_from_gaussian_tuples(
-        self, tuples, a=None, r=None, use_errors=True, use_widths=True, no_limits=False
+    def mapper_from_prior_means(
+        self, means, a=None, r=None, no_limits=False
     ):
         """
         The widths of the new priors are taken from the
@@ -916,21 +916,15 @@ def mapper_from_gaussian_tuples(
         If r is not None then all priors are created with a relative width of r.
         Parameters
         ----------
+        means
+            The median PDF value of every Gaussian, which centres each `GaussianPrior`.
         no_limits
             If `True` generated priors have infinite limits
         r
             The relative width to be assigned to gaussian priors
         a
             print(tuples[i][1], width)
             The absolute width to be assigned to gaussian priors
-        use_errors
-            If True, the passed errors of the model components estimated in a previous `NonLinearSearch` (computed
-            at the prior_passer.sigma value) are used to set the pass Gaussian Prior sigma value (if both width and
-            passed errors are used, the maximum of these two values are used).
-        use_widths
-            If True, the minimum prior widths specified in the prior configs of the model components are used to
-            set the passed Gaussian Prior sigma value (if both widths and passed errors are used, the maximum of
-            these two values are used).
         tuples
             A list of tuples each containing the mean and width of a prior
         Returns
@@ -946,7 +940,6 @@ def mapper_from_gaussian_tuples(
         for i, prior_tuple in enumerate(prior_tuples):
             prior = prior_tuple.prior
             cls = prior_class_dict[prior]
-            mean, sigma = tuples[i]
 
             name = prior_tuple.name
             # Use the name of the collection for configuration when a prior's name
@@ -966,9 +959,9 @@ def mapper_from_gaussian_tuples(
             if a is not None:
                 width = a
             elif r is not None:
-                width = r * mean
+                width = r * means[i]
             else:
-                width = width_modifier(mean)
+                width = width_modifier(means[i])
 
             if no_limits:
                 limits = (float("-inf"), float("inf"))
@@ -978,19 +971,9 @@ def mapper_from_gaussian_tuples(
                 except ConfigException:
                     limits = prior.limits
 
-            if use_errors and not use_widths:
-                sigma = tuples[i][1]
-            elif not use_errors and use_widths:
-                sigma = width
-            elif use_errors and use_widths:
-                sigma = max(tuples[i][1], width)
-            else:
-                raise exc.PriorException(
-                    "use_passed_errors and use_widths are both False, meaning there is no "
-                    "way to pass priors to set up the new model's Gaussian Priors."
-                )
+            sigma = width
 
-            new_prior = GaussianPrior(mean, sigma, *limits)
+            new_prior = GaussianPrior(means[i], sigma, *limits)
             new_prior.id = prior.id
             new_prior.width_modifier = prior.width_modifier
             arguments[prior] = new_prior

autofit/non_linear/grid/simple_grid.py

@@ -54,7 +54,6 @@ def fit(
             samples=MockSamples(
                 max_log_likelihood_instance=best_instance,
                 log_likelihood_list=likelihoods,
-                gaussian_tuples=None
             ),
             model=model
         )

autofit/non_linear/mock/mock_result.py

@@ -21,7 +21,7 @@ def __init__(
             max_log_likelihood_instance=self.instance, model=model or ModelMapper()
         )
 
-        self.gaussian_tuples = None
+        self.prior_means = None
         self.analysis = analysis
         self.search = search
         self.model = model

autofit/non_linear/mock/mock_samples.py

@@ -16,7 +16,7 @@ def __init__(
         samples_info=None,
         max_log_likelihood_instance=None,
         log_likelihood_list=None,
-        gaussian_tuples=None,
+        prior_means=None,
         **kwargs,
     ):
         self._log_likelihood_list = log_likelihood_list
@@ -35,7 +35,7 @@ def __init__(
         )
 
         self._max_log_likelihood_instance = max_log_likelihood_instance
-        self._gaussian_tuples = gaussian_tuples
+        self._prior_means = prior_means
 
     @property
     def default_sample_list(self):
@@ -65,11 +65,12 @@ def max_log_likelihood(self, as_instance: bool = True):
 
         return self._max_log_likelihood_instance
 
-    def gaussian_priors_at_sigma(self, sigma=None):
-        if self._gaussian_tuples is None:
-            return super().gaussian_priors_at_sigma(sigma=sigma)
+    @property
+    def prior_means(self):
+        if self._prior_means is None:
+            return super().prior_means
 
-        return self._gaussian_tuples
+        return self._prior_means
 
     @property
     def unconverged_sample_size(self):

autofit/non_linear/mock/mock_search.py

@@ -121,9 +121,8 @@ def _fit(self, model, analysis):
                 self.sample_multiplier * fit, _make_samples(model)
             ),
             model=model,
-            gaussian_tuples=[
-                (prior.mean, prior.width if math.isfinite(prior.width) else 1.0)
-                for prior in sorted(model.priors, key=lambda prior: prior.id)
+            prior_means=[
+                prior.mean for prior in sorted(model.priors, key=lambda prior: prior.id)
             ],
         )
 
@@ -142,9 +141,8 @@ def perform_update(self, model, analysis, during_analysis, search_internal=None)
             sample_list=samples_with_log_likelihood_list(
                 [1.0, 2.0], _make_samples(model)
             ),
-            gaussian_tuples=[
-                (prior.mean, prior.width if math.isfinite(prior.width) else 1.0)
-                for prior in sorted(model.priors, key=lambda prior: prior.id)
+            prior_means=[
+                prior.mean for prior in sorted(model.priors, key=lambda prior: prior.id)
             ],
             model=model,
         )

autofit/non_linear/paths/abstract.py

@@ -393,6 +393,9 @@ def save_search_internal(self, obj):
     def load_search_internal(self):
         raise NotImplementedError
 
+    def remove_search_internal(self):
+        raise NotImplementedError
+
     @property
     @abstractmethod
     def is_complete(self) -> bool:

autofit/non_linear/paths/database.py

@@ -226,6 +226,9 @@ def save_search_internal(self, obj):
     def load_search_internal(self):
         pass
 
+    def remove_search_internal(self):
+        pass
+
     @property
     def fit(self) -> Fit:
         if self._fit is None:

autofit/non_linear/paths/directory.py

@@ -1,3 +1,5 @@
+import shutil
+
 import dill
 import json
 import os
@@ -206,6 +208,18 @@ def load_search_internal(self):
         with open_(filename, "rb") as f:
             return dill.load(f)
 
+    def remove_search_internal(self):
+        """
+        Remove the internal representation of a non-linear search.
+
+        This deletes the entire `search_internal` folder, including a .pickle / .dill file containing the interal
+        results and files with the timer values.
+
+        This folder can often have a large filesize, thus deleting it can reduce hard-disk use of the model-fit.
+        """
+        shutil.rmtree(self.search_internal_path)
+        os.rmdir(self.search_internal_path)
+
     def completed(self):
         """
         Mark the search as complete by saving a file

autofit/non_linear/paths/null.py

@@ -87,6 +87,9 @@ def save_search_internal(self, obj):
     def load_search_internal(self):
         pass
 
+    def remove_search_internal(self):
+        pass
+
     @property
     def search_internal_path(self):
         pass

autofit/non_linear/result.py

@@ -220,14 +220,13 @@ def projected_model(self) -> AbstractPriorModel:
 
     @property
     def model(self):
-        use_errors = conf.instance["general"]["prior_passer"]["use_errors"]
-        use_widths = conf.instance["general"]["prior_passer"]["use_widths"]
 
         if self.__model is None:
-            tuples = self.samples.gaussian_priors_at_sigma(sigma=self.sigma)
-            self.__model = self.samples.model.mapper_from_gaussian_tuples(
-                tuples, use_errors=use_errors, use_widths=use_widths
+
+            self.__model = self.samples.model.mapper_from_prior_means(
+                means=self.samples.prior_means
             )
+
         return self.__model
 
     @model.setter

autofit/non_linear/samples/interface.py

@@ -134,8 +134,8 @@ def model_absolute(self, a: float) -> AbstractPriorModel:
         A model mapper created by taking results from this search and creating priors with the defined absolute
         width.
         """
-        return self.model.mapper_from_gaussian_tuples(
-            self.gaussian_priors_at_sigma(sigma=self.sigma), a=a
+        return self.model.mapper_from_prior_means(
+            means=self.prior_means, a=a
         )
 
     def model_relative(self, r: float) -> AbstractPriorModel:
@@ -159,8 +159,8 @@ def model_relative(self, r: float) -> AbstractPriorModel:
         A model mapper created by taking results from this search and creating priors with the defined relative
         width.
         """
-        return self.model.mapper_from_gaussian_tuples(
-            self.gaussian_priors_at_sigma(sigma=self.sigma), r=r
+        return self.model.mapper_from_prior_means(
+            means=self.prior_means, r=r
         )
 
     def model_bounded(self, b: float) -> AbstractPriorModel:
@@ -187,31 +187,21 @@ def model_bounded(self, b: float) -> AbstractPriorModel:
             floats=self.max_log_likelihood(as_instance=False), b=b
         )
 
-    @property
-    def sigma(self):
-        return conf.instance["general"]["prior_passer"]["sigma"]
+    def _instance_from_vector(self, vector: List[float]) -> ModelInstance:
+        return self.model.instance_from_vector(vector=vector, ignore_prior_limits=True)
 
-    def gaussian_priors_at_sigma(self, sigma: float) -> [List]:
+    @property
+    def prior_means(self) -> [List]:
         """
-        Returns `GaussianPrior`'s of every parameter in a fit for use with non-linear search chaining.
-
-        `GaussianPrior`s of every parameter used to link its inferred values and errors to priors used to sample the
+        The mean of every parameter used to link its inferred values and errors to priors used to sample the
         same (or similar) parameters in a subsequent search, where:
 
-        - The mean is given by maximum log likelihood model values.
-        - Their errors are omitted, as this information is not available from an search. When these priors are
-          used to link to another search, it will thus automatically use the prior config values.
-
-        Parameters
-        ----------
-        sigma
-            The sigma limit within which the PDF is used to estimate errors (e.g. sigma = 1.0 uses 0.6826 of the PDF).
+        - The mean is given by their most-probable values median PDF values if using a sampler that provides this
+          information(using median_pdf(as_instance=False)).
+        - The man is given by the maximum log likelihood values otherwise (e.g. for a maximum likelihood estimator).
         """
-        return list(
-            map(
-                lambda vector: (vector, 0.0), self.max_log_likelihood(as_instance=False)
-            )
-        )
 
-    def _instance_from_vector(self, vector: List[float]) -> ModelInstance:
-        return self.model.instance_from_vector(vector=vector, ignore_prior_limits=True)
+        try:
+            return self.median_pdf(as_instance=False)
+        except AttributeError:
+            return self.max_log_likelihood(as_instance=False)