{ "cells": [ { "cell_type": "markdown", "id": "9e96adc8-68d1-45d5-ab1e-15a3a72ae637", "metadata": {}, "source": [ "# CALCUL AND SAVE SST STATS" ] }, { "cell_type": "code", "execution_count": 2, "id": "1aac342c-fb07-4020-9f63-b185496abe27", "metadata": {}, "outputs": [], "source": [ "import numpy as N\n", "import xarray as xr\n", "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import sys\n", "import dask\n", "from os import path\n", "from dask.distributed import Client, LocalCluster, progress\n", "import time\n", "from tools import my_percentile\n", "from scipy.optimize import least_squares,curve_fit\n", "import dask_hpcconfig\n", "from dask_jobqueue import PBSCluster\n", "import glob" ] }, { "cell_type": "markdown", "id": "9064ab92-5d78-45a6-a6f8-2251f8eb3b9a", "metadata": { "tags": [] }, "source": [ "### conseils dask\n", "\n", "pour Datarmor il vaut mieux avoir 1 thread/worker\n", "il vaut mieux avoir des chunks assez fa" ] }, { "cell_type": "code", "execution_count": 3, "id": "293e8282-f4c5-42e5-a5ec-602afa308123", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home1/datahome/mcaillau/conda-env/pydask3/lib/python3.10/site-packages/dask_jobqueue/core.py:255: FutureWarning: job_extra has been renamed to job_extra_directives. You are still using it (even if only set to []; please also check config files). If you did not set job_extra_directives yet, job_extra will be respected for now, but it will be removed in a future release. If you already set job_extra_directives, job_extra is ignored and you can remove it.\n", " warnings.warn(warn, FutureWarning)\n", "/home1/datahome/mcaillau/conda-env/pydask3/lib/python3.10/site-packages/dask_jobqueue/pbs.py:82: FutureWarning: project has been renamed to account as this kwarg was used wit -A option. You are still using it (please also check config files). If you did not set account yet, project will be respected for now, but it will be removed in a future release. If you already set account, project is ignored and you can remove it.\n", " warnings.warn(warn, FutureWarning)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "http://10.148.1.73:8787/status\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/home1/datahome/mcaillau/conda-env/pydask3/lib/python3.10/site-packages/dask_jobqueue/core.py:255: FutureWarning: job_extra has been renamed to job_extra_directives. You are still using it (even if only set to []; please also check config files). If you did not set job_extra_directives yet, job_extra will be respected for now, but it will be removed in a future release. If you already set job_extra_directives, job_extra is ignored and you can remove it.\n", " warnings.warn(warn, FutureWarning)\n", "/home1/datahome/mcaillau/conda-env/pydask3/lib/python3.10/site-packages/dask_jobqueue/pbs.py:82: FutureWarning: project has been renamed to account as this kwarg was used wit -A option. You are still using it (please also check config files). If you did not set account yet, project will be respected for now, but it will be removed in a future release. If you already set account, project is ignored and you can remove it.\n", " warnings.warn(warn, FutureWarning)\n" ] } ], "source": [ "#pour l'ensemble des pas de temps\n", "#cluster = PBSCluster(processes=3,cores=9) #16min 44\n", "cluster = PBSCluster(processes=3,cores=3) #16min 44\n", "\n", "cluster.scale(jobs=4)\n", "print(cluster.dashboard_link)" ] }, { "cell_type": "code", "execution_count": 4, "id": "7d54b8c0-7f90-4d6f-b55a-253be3db0568", "metadata": {}, "outputs": [], "source": [ "# explicitly connect to the cluster we just created\n", "client = Client(cluster)" ] }, { "cell_type": "code", "execution_count": 5, "id": "385847a6-d04f-40af-9538-caa1b8448669", "metadata": {}, "outputs": [], "source": [ "#read grid\n", "ds_grid=xr.open_dataset('/home/shom_simuref/CROCO/ODC/CONFIGS/MEDITERRANEE_GLOBALE/CROCO_FILES/test2.nc')" ] }, { "cell_type": "code", "execution_count": 6, "id": "9ad6f687-3c5f-47b6-9f41-96f05d47808b", "metadata": {}, "outputs": [], "source": [ "path=\"/home/shom_simuref/CROCO/ODC/SIMU-RESULT/HINDCAST_2012_2013/OUTPUTS_201207_201307/\"" ] }, { "cell_type": "code", "execution_count": 7, "id": "d2ac1911-9fb8-4e41-803e-ae16d4600de3", "metadata": {}, "outputs": [], "source": [ "list_model=glob.glob(path+\"croco_his_surf_2*.nc\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "53ea47d2-001f-42fb-b5f4-3b122d4e8551", "metadata": {}, "outputs": [], "source": [ "chunks={'time':121,'xi_rho':315,'eta_rho':878}\n", "chunks={'time':800,'s_rho':-1,'xi_rho':865,'eta_rho':936}\n", "date_start=\"2012-06-01\"\n", "date_end=\"2013-08-01\"\n", "time_range=slice(date_start,date_end)" ] }, { "cell_type": "code", "execution_count": 9, "id": "fc3b2c57-44a8-4c91-a0b6-e0b452e26694", "metadata": {}, "outputs": [], "source": [ "#store data dir\n", "stat_dir='/home/shom_simuref/CROCO/ODC/POSTPROC/SST/'" ] }, { "cell_type": "code", "execution_count": 10, "id": "ae5157d2-511e-465e-a2a2-d29b72eeade7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "data size in GB 425.54\n", "\n" ] } ], "source": [ "ds=xr.open_mfdataset(list_model, parallel=False,chunks=\"auto\",\n", " concat_dim=\"time\", combine=\"nested\",\n", " data_vars='minimal', coords='minimal', compat='override')\n", "# preprocess=partial_func,data_vars='minimal', coords='minimal', compat='override')\n", "\n", "print('data size in GB {:0.2f}\\n'.format(ds.nbytes / 1e9))\n" ] }, { "cell_type": "code", "execution_count": 11, "id": "4738f6a4-f7ec-4ad7-818f-46e51be68685", "metadata": {}, "outputs": [], "source": [ "#ds=ds.isel(time=slice(0,-1,12))" ] }, { "cell_type": "code", "execution_count": 12, "id": "982c01c8-90b3-4a8f-b192-c3bdd6d6e107", "metadata": {}, "outputs": [], "source": [ "#because of MPINOLAND \n", "for var in [\"xi_rho\",\"eta_rho\",\"xi_u\",\"eta_v\",\"lon_rho\",\"lat_rho\",\"lon_u\",\"lat_v\"]:\n", " ds[var]=ds_grid[var]" ] }, { "cell_type": "code", "execution_count": 13, "id": "508a1eca-ad12-4225-b7eb-368e473fc4d4", "metadata": {}, "outputs": [], "source": [ "#remove duplicates\n", "_,index=N.unique(ds.time,return_index=True)\n", "ds=ds.isel(time=index)" ] }, { "cell_type": "code", "execution_count": 14, "id": "a8b631fc-8332-40a0-a6d7-5670ffacce7d", "metadata": {}, "outputs": [], "source": [ "#select time range\n", "ds=ds.sel(time=time_range)" ] }, { "cell_type": "code", "execution_count": 15, "id": "0e85cd68-226d-4cb1-a004-d0f277c01020", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'temp' (time: 8760, eta_rho: 936, xi_rho: 2595)>\n",
       "dask.array<rechunk-merge, shape=(8760, 936, 2595), dtype=float32, chunksize=(800, 936, 865), chunktype=numpy.ndarray>\n",
       "Coordinates:\n",
       "  * xi_rho   (xi_rho) int64 0 1 2 3 4 5 6 ... 2588 2589 2590 2591 2592 2593 2594\n",
       "  * eta_rho  (eta_rho) int64 0 1 2 3 4 5 6 7 ... 928 929 930 931 932 933 934 935\n",
       "  * time     (time) datetime64[ns] 2012-07-01T13:00:00 ... 2013-07-01T12:00:00\n",
       "    lon_rho  (eta_rho, xi_rho) float64 dask.array<chunksize=(936, 865), meta=np.ndarray>\n",
       "    lat_rho  (eta_rho, xi_rho) float64 dask.array<chunksize=(936, 865), meta=np.ndarray>\n",
       "Attributes:\n",
       "    long_name:  SST\n",
       "    units:      Celsius\n",
       "    field:
" ], "text/plain": [ "\n", "dask.array\n", "Coordinates:\n", " * xi_rho (xi_rho) int64 0 1 2 3 4 5 6 ... 2588 2589 2590 2591 2592 2593 2594\n", " * eta_rho (eta_rho) int64 0 1 2 3 4 5 6 7 ... 928 929 930 931 932 933 934 935\n", " * time (time) datetime64[ns] 2012-07-01T13:00:00 ... 2013-07-01T12:00:00\n", " lon_rho (eta_rho, xi_rho) float64 dask.array\n", " lat_rho (eta_rho, xi_rho) float64 dask.array\n", "Attributes:\n", " long_name: SST\n", " units: Celsius\n", " field: " ] }, "execution_count": 15, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ds=ds.chunk(chunks)\n", "ds.temp" ] }, { "cell_type": "code", "execution_count": 16, "id": "9bf4fc5e-dc7d-4c90-a35a-717ccb01ab5c", "metadata": {}, "outputs": [], "source": [ "coord_dict={\"xi_rho\":\"X\",\"eta_rho\":\"Y\",\"xi_u\":\"X_U\",\"eta_v\":\"Y_V\"}\n", "ds=ds.assign_coords({\"X\":ds_grid.lon_rho[0,:], \"Y\":ds_grid.lat_rho[:,0]})\n", "ds2=ds.swap_dims(coord_dict)" ] }, { "cell_type": "code", "execution_count": 17, "id": "1581cfcf-38da-481a-bd87-96e449babcb4", "metadata": {}, "outputs": [], "source": [ "sst=ds2.temp" ] }, { "cell_type": "code", "execution_count": 18, "id": "3b132ad5-1e28-442c-8165-655993a77863", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
<xarray.DataArray 'temp' (time: 8760, Y: 936, X: 2595)>\n",
       "dask.array<rechunk-merge, shape=(8760, 936, 2595), dtype=float32, chunksize=(800, 936, 865), chunktype=numpy.ndarray>\n",
       "Coordinates:\n",
       "    xi_rho   (X) int64 0 1 2 3 4 5 6 7 ... 2588 2589 2590 2591 2592 2593 2594\n",
       "    eta_rho  (Y) int64 0 1 2 3 4 5 6 7 8 ... 927 928 929 930 931 932 933 934 935\n",
       "  * time     (time) datetime64[ns] 2012-07-01T13:00:00 ... 2013-07-01T12:00:00\n",
       "    lon_rho  (Y, X) float64 dask.array<chunksize=(936, 865), meta=np.ndarray>\n",
       "    lat_rho  (Y, X) float64 dask.array<chunksize=(936, 865), meta=np.ndarray>\n",
       "  * X        (X) float64 -7.0 -6.983 -6.967 -6.95 ... 36.18 36.2 36.22 36.23\n",
       "  * Y        (Y) float64 30.23 30.25 30.27 30.28 30.3 ... 45.77 45.78 45.8 45.82\n",
       "Attributes:\n",
       "    long_name:  SST\n",
       "    units:      Celsius\n",
       "    field:
" ], "text/plain": [ "\n", "dask.array\n", "Coordinates:\n", " xi_rho (X) int64 0 1 2 3 4 5 6 7 ... 2588 2589 2590 2591 2592 2593 2594\n", " eta_rho (Y) int64 0 1 2 3 4 5 6 7 8 ... 927 928 929 930 931 932 933 934 935\n", " * time (time) datetime64[ns] 2012-07-01T13:00:00 ... 2013-07-01T12:00:00\n", " lon_rho (Y, X) float64 dask.array\n", " lat_rho (Y, X) float64 dask.array\n", " * X (X) float64 -7.0 -6.983 -6.967 -6.95 ... 36.18 36.2 36.22 36.23\n", " * Y (Y) float64 30.23 30.25 30.27 30.28 30.3 ... 45.77 45.78 45.8 45.82\n", "Attributes:\n", " long_name: SST\n", " units: Celsius\n", " field: " ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sst" ] }, { "cell_type": "markdown", "id": "d78f4ca1-df9f-43ef-989d-56a2613a04e1", "metadata": {}, "source": [ "## MONTHLY MEAN" ] }, { "cell_type": "code", "execution_count": 79, "id": "69aebda7-213c-4a4a-8f0b-133e1104f3b3", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "compute mean\n", "CPU times: user 18.9 s, sys: 2.13 s, total: 21.1 s\n", "Wall time: 4min 12s\n" ] } ], "source": [ "%%time\n", "#4min12s on 4 workers and 3 proc par worker\n", "sst=ds2.temp\n", "print('compute mean')\n", "sst_mean = sst.resample(time=\"1MS\").mean().compute()\n", "sst_mean=sst_mean.assign_attrs({\"start_date\":date_start,\"end_date\":date_end})\n", "#write results for faster treatment\n", "sst_mean.to_netcdf(f'{stat_dir}/mean_sst_croco.nc',mode='w')" ] }, { "cell_type": "markdown", "id": "920fc575-e21f-42f5-ba6c-aa782d829c84", "metadata": {}, "source": [ "### DAILY MEAN" ] }, { "cell_type": "code", "execution_count": 19, "id": "1fcc2dde-e226-4001-a2f7-310a705cdd02", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "compute daily mean\n", "CPU times: user 39.2 s, sys: 9.66 s, total: 48.8 s\n", "Wall time: 6min 31s\n" ] } ], "source": [ "%%time\n", "#4min12s on 4 workers and 3 proc par worker\n", "sst=ds2.temp\n", "print('compute daily mean')\n", "sst_daily_mean = sst.resample(time=\"1D\").mean().compute()\n", "sst_daily_mean=sst_daily_mean.assign_attrs({\"start_date\":date_start,\"end_date\":date_end})\n", "#write results for faster treatment\n", "sst_daily_mean.to_netcdf(f'{stat_dir}/mean_daily_sst_croco.nc',mode='w')" ] }, { "cell_type": "markdown", "id": "8e7105cb-001e-46ce-a977-8cd2fd3b50d5", "metadata": {}, "source": [ "## PERCENTILES" ] }, { "cell_type": "code", "execution_count": 19, "id": "7c669ecc-0752-46eb-a8be-d8abca75307b", "metadata": {}, "outputs": [], "source": [ "from tools import my_percentile" ] }, { "cell_type": "markdown", "id": "ae853037-8921-4286-b280-ba0a3770e74a", "metadata": {}, "source": [ "#### re chunks to percentile " ] }, { "cell_type": "code", "execution_count": 20, "id": "156b2e06-26ec-4a00-ad74-168571fffaf9", "metadata": {}, "outputs": [], "source": [ "#sst=sst.chunk({'time':-1,'Y':52,'X':173})" ] }, { "cell_type": "code", "execution_count": 21, "id": "ff43be21-2286-40cf-9045-ed745bb6ac8e", "metadata": {}, "outputs": [], "source": [ "#%%time\n", "#per95=my_percentile(sst,0.95,\"time\")\n", "#per95=per95.compute()\n", "#per95.to_netcdf(f'{stat_dir}/sst_percentile_95.nc',mode='w')" ] }, { "cell_type": "code", "execution_count": 22, "id": "891f67ad-cb04-4d25-8fa0-51e650250d9b", "metadata": {}, "outputs": [], "source": [ "#%%time\n", "#per5=my_percentile(sst,0.05,\"time\")\n", "#per5=per5.compute()\n", "#per5.to_netcdf(f'{stat_dir}/sst_percentile_5.nc',mode='w')" ] }, { "cell_type": "markdown", "id": "1ff0d055-84a8-4028-8844-666a50c43342", "metadata": {}, "source": [ "### SEVIRI" ] }, { "cell_type": "code", "execution_count": 20, "id": "30b29c59-e8d6-488e-a9d0-0ba81ca9faf3", "metadata": {}, "outputs": [], "source": [ "seviri_path=\"/home/shom_simuref/CROCO/ODC/DATA/SST_SEVIRI/\"" ] }, { "cell_type": "code", "execution_count": 21, "id": "ac5f04ac-c9a9-448f-8b11-6ceb67858a6d", "metadata": {}, "outputs": [], "source": [ "chunks_sevi={'time':444,\"lon\":-1,\"lat\":-1}\n" ] }, { "cell_type": "code", "execution_count": 22, "id": "81813433-80c1-47b5-bc13-0333741a3cd5", "metadata": {}, "outputs": [], "source": [ "ds_sevi=xr.open_mfdataset(glob.glob(seviri_path+'201[2-3]*.nc'),chunks=chunks_sevi,\n", " combine=\"by_coords\",\n", " data_vars='minimal', coords='minimal', compat='override')" ] }, { "cell_type": "markdown", "id": "0ce0aeb5-2e62-4560-a979-189114b46f6a", "metadata": {}, "source": [ "### reduce to croco time bnds" ] }, { "cell_type": "code", "execution_count": 23, "id": "64c9ffc4-fcaf-4be2-a691-abe7b68d5339", "metadata": {}, "outputs": [], "source": [ "time_bnds=slice(ds.time[0],ds.time[-1])" ] }, { "cell_type": "code", "execution_count": 24, "id": "d29f4b71-457b-4631-9929-50864993982f", "metadata": {}, "outputs": [], "source": [ "ds_sevi=ds_sevi.sel(time=time_bnds)" ] }, { "cell_type": "code", "execution_count": 25, "id": "afff4a87-cbc9-448a-adba-29454ddc2ca0", "metadata": {}, "outputs": [], "source": [ "sst_sevi=ds_sevi.sea_surface_temperature-273.15" ] }, { "cell_type": "markdown", "id": "747f56e9-afd6-4c51-be66-2e209f160f42", "metadata": {}, "source": [ "### monthly mean" ] }, { "cell_type": "code", "execution_count": 26, "id": "4334b7e3-317e-426f-8eba-76d9a893a6a5", "metadata": {}, "outputs": [], "source": [ "sst_sevi_mean=sst_sevi.resample(time='1MS').mean()" ] }, { "cell_type": "code", "execution_count": 28, "id": "13e9a6ae-6623-4c2b-a375-c1dcc71e42a7", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 20 ms, sys: 0 ns, total: 20 ms\n", "Wall time: 59.5 ms\n" ] } ], "source": [ "%%time\n", "sst_sevi_mean=sst_sevi_mean.compute()\n", "sst_sevi=sst_sevi.assign_attrs({\"start_date\":date_start,\"end_date\":date_end})\n", "sst_sevi_mean.to_netcdf(f'{stat_dir}/mean_sst_sevi.nc',mode=\"w\")" ] }, { "cell_type": "markdown", "id": "2cbbeec0-14fd-4b60-a2b2-f8dae1e58cce", "metadata": {}, "source": [ "### daily mean" ] }, { "cell_type": "code", "execution_count": 29, "id": "9ec0bc83-e78b-4e58-b9c9-a589e44b5487", "metadata": {}, "outputs": [], "source": [ "sst_sevi_daily_mean=sst_sevi.resample(time='1D').mean()\n", "sst_sevi_daily_mean=sst_sevi_daily_mean.assign_attrs({\"start_date\":date_start,\"end_date\":date_end})\n", "sst_sevi_daily_mean.to_netcdf(f'{stat_dir}/mean_daily_sst_sevi.nc',mode=\"w\")" ] }, { "cell_type": "markdown", "id": "3e0e7ecb-9b74-44e5-8b0c-d1647c316f11", "metadata": {}, "source": [ "## percentiles" ] }, { "cell_type": "markdown", "id": "81c3ce16-3c6b-454a-a8df-acdd04bb10ce", "metadata": {}, "source": [ "### re chunk for PERCENTILE" ] }, { "cell_type": "code", "execution_count": 31, "id": "565ba86a-824c-46cc-8786-a963e20a33aa", "metadata": {}, "outputs": [], "source": [ "sst_sevi=sst_sevi.chunk({'time':-1,\"lat\":-1,\"lon\":-1})" ] }, { "cell_type": "code", "execution_count": 32, "id": "177604e4-e459-4643-aff0-c07c0fa05afa", "metadata": {}, "outputs": [], "source": [ "sst_sevi2=sst_sevi.interpolate_na(method=\"linear\",limit=None,max_gap=None,dim=\"time\")" ] }, { "cell_type": "code", "execution_count": 33, "id": "ce29f127-c371-41ca-8750-a4854844c405", "metadata": {}, "outputs": [], "source": [ "sst_sevi2=sst_sevi2.ffill('time')\n", "sst_sevi2=sst_sevi2.bfill('time')\n" ] }, { "cell_type": "code", "execution_count": 34, "id": "9a7d4296-8336-4215-b9dd-d862a057122c", "metadata": { "tags": [] }, "outputs": [], "source": [ "#%%time\n", "##35 sec sur 4 jobs de 6 processes et 12 cores ( 2 threads/worker)\n", "#sst_sevi_p95=my_percentile(sst_sevi2,0.95,\"time\").compute()\n", "#sst_sevi_p95.to_netcdf(f'{stat_dir}/sst_sevi_p95.nc',mode=\"w\")" ] }, { "cell_type": "code", "execution_count": 35, "id": "59af4508-d149-4eaa-9122-20e9d692abcd", "metadata": {}, "outputs": [], "source": [ "#%%time\n", "#sst_sevi_p5=my_percentile(sst_sevi2,0.05,\"time\").compute()\n", "#sst_sevi_p5.to_netcdf(f'{stat_dir}/sst_sevi_p5.nc',mode=\"w\")" ] }, { "cell_type": "markdown", "id": "f1cb0707-800b-4b16-9f44-7440532cc199", "metadata": { "tags": [] }, "source": [ "## RMSE CROCO-SEVIRI" ] }, { "cell_type": "markdown", "id": "3bdf709d-2b74-4639-ac27-76ae77d6c7f2", "metadata": {}, "source": [ "### First compute seviri" ] }, { "cell_type": "code", "execution_count": 36, "id": "587b5301-08f8-4b6a-ab12-ae3a9a557045", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 5.1 s, sys: 3.95 s, total: 9.05 s\n", "Wall time: 1min 21s\n" ] } ], "source": [ "%%time\n", "#1min 21s sur 4 workers avec 3 procs par worker\n", "sst_sevi2=sst_sevi2.compute()" ] }, { "cell_type": "markdown", "id": "51652893-d7e2-485f-b4bd-7930e6c0e694", "metadata": {}, "source": [ "### interp croco on seviri horizontal grid" ] }, { "cell_type": "code", "execution_count": 37, "id": "46a7b7f0-0985-4453-8c25-c67706436f62", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 36 ms, sys: 0 ns, total: 36 ms\n", "Wall time: 115 ms\n" ] } ], "source": [ "sst_croco=sst.interp(X=sst_sevi.lon,Y=sst_sevi.lat)" ] }, { "cell_type": "code", "execution_count": 38, "id": "37572b92-e75e-4a1c-a60a-cf223d5b61bc", "metadata": {}, "outputs": [], "source": [ "sst_croco=sst_croco.chunk({'time':2000})" ] }, { "cell_type": "code", "execution_count": 39, "id": "566fbfc0-d9e3-46cc-b9cb-736782a1a320", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 41.7 s, sys: 35.6 s, total: 1min 17s\n", "Wall time: 6min 23s\n" ] } ], "source": [ "%%time\n", "#6 min 23s sur 4 workers avec 3 procs par worker\n", "sst_croco=sst_croco.compute()" ] }, { "cell_type": "markdown", "id": "a5c99c2e-ff7e-40d1-9d96-89a45198dd8b", "metadata": {}, "source": [ "### interp croco on seviri time grid" ] }, { "cell_type": "code", "execution_count": 40, "id": "a06f864c-1605-4cad-bb54-e34a40bb8eb0", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 16.2 s, sys: 25.8 s, total: 42 s\n", "Wall time: 40.1 s\n" ] } ], "source": [ "%%time\n", "#40 s sur 4 workers avec 3 procs par worker\n", "sst_croco2=sst_croco.interp(time=sst_sevi2.time).compute()\n" ] }, { "cell_type": "markdown", "id": "de7aa030-acc5-4668-869e-686c61e68dc1", "metadata": {}, "source": [ "## Compute the RMSE" ] }, { "cell_type": "code", "execution_count": 47, "id": "9814a354-9445-43fb-9ce8-baf617b9590c", "metadata": {}, "outputs": [], "source": [ "import xskillscore as xs" ] }, { "cell_type": "markdown", "id": "42744107-ad4a-48ce-8c57-c283e404e4b1", "metadata": {}, "source": [ "### calcul of monthly rmse" ] }, { "cell_type": "code", "execution_count": 48, "id": "c5311f5d-c915-4760-96a3-a255a3e22442", "metadata": { "tags": [] }, "outputs": [], "source": [ "ds_merge=xr.merge([sst_croco2.rename('croco'),sst_sevi2.rename('seviri')])" ] }, { "cell_type": "code", "execution_count": 56, "id": "8d90aded-5c08-4bc0-81d5-600c3697c392", "metadata": {}, "outputs": [], "source": [ "#custom function to use with resample in xarray\n", "def calc_rmse(data):\n", " return xs.rmse(data.croco,data.seviri,dim=\"time\",skipna=True)" ] }, { "cell_type": "code", "execution_count": 50, "id": "399c1dd8-5b95-4356-9e90-a383b670beca", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 13.1 s, sys: 9.64 s, total: 22.7 s\n", "Wall time: 22 s\n" ] } ], "source": [ "%%time\n", "#22s sur 4 workers avec 3 procs par worker\n", "rmse_month=ds_merge.resample(time='1MS').apply(calc_rmse).compute()" ] }, { "cell_type": "code", "execution_count": 65, "id": "5cbd6ea3-9d2c-4f59-a968-e166c951b594", "metadata": {}, "outputs": [], "source": [ "rmse_month=rmse_month.assign_attrs({\"start_date\":date_start,\"end_date\":date_end})\n", "rmse_month=rmse_month.rename('rmse')\n", "rmse_month.to_netcdf(f'{stat_dir}/rmse_croco_sevi_monthly.nc',mode='w')" ] }, { "cell_type": "markdown", "id": "c21d6622-9ae5-49f1-aa46-0390f1f26c8b", "metadata": {}, "source": [ "### calcul of annual rmse" ] }, { "cell_type": "code", "execution_count": 64, "id": "c01af437-2d51-48da-9992-a48f9f88b0ef", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "CPU times: user 23.8 s, sys: 9.01 s, total: 32.8 s\n", "Wall time: 31.8 s\n" ] } ], "source": [ "%%time\n", "#31s sur 4 workers avec 3 procs par worker\n", "rmse=xs.rmse(sst_croco2,sst_sevi2,dim=\"time\",skipna=True)\n", "rmse=rmse.compute()\n", "rmse=rmse.rename('rmse')\n", "rmse=rmse.assign_attrs({\"start_date\":date_start,\"end_date\":date_end})\n", "rmse.to_netcdf(f'{stat_dir}/rmse_croco_sevi_annual.nc',mode='w')\n" ] }, { "cell_type": "code", "execution_count": 61, "id": "de0bd018-949e-4052-a2e8-05aebaf4edf3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "996a545b-35d5-4d19-9131-603256004a8d", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.8" } }, "nbformat": 4, "nbformat_minor": 5 }