diff --git a/notebooks/Tutorial/calversion.ipynb b/notebooks/Tutorial/calversion.ipynb index 2ae7833374acabdc1cb83a5acf10085241849524..14933bab4ff9399154c657de68c43ae2f98dac2e 100644 --- a/notebooks/Tutorial/calversion.ipynb +++ b/notebooks/Tutorial/calversion.ipynb @@ -10,7 +10,29 @@ "\n", "A small example how to adapt a notebook to run with the offline calibration package \"pycalibation\".\n", "\n", - "The first cell contains all parameters that should be exposed to the command line." + "The first cell contains all parameters that should be exposed to the command line.\n", + "\n", + "To run this notebooks with several different input parameters in parallel by submitting multiple slurm jobs, for example for various random seed we can do the following:\n", + "\n", + "xfel-calibrate TUTORIAL TEST --random-seed 1,2,3,4\n", + "\n", + "or\n", + "\n", + "xfel-calibrate TUTORIAL TEST --random-seed 1-5\n", + "\n", + "will produce 4 jobs:\n", + "\n", + "Parsed input 1,2,3,4 to [1, 2, 3, 4]\n", + "\n", + "Submitted job: 1169340\n", + "\n", + "Submitted job: 1169341\n", + "\n", + "Submitted job: 1169342\n", + "\n", + "Submitted job: 1169343\n", + "\n", + "Submitted the following SLURM jobs: 1169340,1169341,1169342,1169343" ] }, { @@ -23,7 +45,7 @@ "source": [ "out_folder = \"/gpfs/exfel/data/scratch/amunnich/tutorial\" # output folder\n", "sensor_size = [10, 30] # defining the picture size\n", - "random_seed = 2345 # random seed for filling of fake data array. Change it to produce different results.\n", + "random_seed = [2345] # random seed for filling of fake data array. Change it to produce different results, range allowed\n", "runs = 500 # how may iterations to fill histograms\n", "cluster_profile = \"tutorial\" " ] @@ -32,7 +54,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "First include what we need and set up the cluster profile. Everything that has a written response in a cell will show up in the report, e.g. prints but also return values or errors." + "First include what we need and set up the cluster profile for parallel processing on one node utilising more than one core.\n", + "Everything that has a written response in a cell will show up in the report, e.g. prints but also return values or errors." ] }, { @@ -105,16 +128,19 @@ }, "outputs": [], "source": [ + "# in order to run several random seeds in parallel the parameter has to be a list. To use the current single value in this \n", + "# notebook we use the first entry in the list\n", + "random_seed_single = random_seed[0]\n", "fake_data = []\n", "for i in range(runs):\n", - " fake_data.append(data_creation(random_seed+10*i))" + " fake_data.append(data_creation(random_seed_single+10*i))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Plot the random image. everything we write here in the markup cells will show up as text in the report." + "Create some random images and plot them. Everything we write here in the markup cells will show up as text in the report." ] }, { @@ -183,7 +209,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To parallelise jobs we use the ipyparallel client." + "To parallelise jobs we use the ipyparallel client. This will run on one node an ipcluster with the specified number of cores given in xfel_calibrate/notebooks.py." ] }, { diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py index a0e78baa11d69a58a3573388ac190f7882218800..43cd62ad8cb1d9fadb609fd12d8423874b1af801 100644 --- a/xfel_calibrate/notebooks.py +++ b/xfel_calibrate/notebooks.py @@ -84,7 +84,7 @@ notebooks = { "TUTORIAL": { "TEST": { "notebook": "notebooks/Tutorial/calversion.ipynb", - "concurrency": {"parameter": None, + "concurrency": {"parameter": "random_seed", "default concurrency": None, "cluster cores": 32}, },