Add TTYPE1=RANGE in RANGE FITS, add options to save V1.0 FITS compati…

…ble S-MOCs
cds-astro · Nov 10, 2022 · 6f87e98 · 6f87e98
1 parent 1b1c6d8
commit 6f87e98
Show file tree

Hide file tree

Showing 20 changed files with 275 additions and 184 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -27,14 +27,14 @@ jobs:
          cd crates/set
          for PYBIN in /opt/python/cp38*/bin; do
            "${PYBIN}/pip" install maturin
-           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
-           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
+           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
+           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
          done
          cd ../cli
          for PYBIN in /opt/python/cp38*/bin; do
            "${PYBIN}/pip" install maturin
-           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
-           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
+           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
+           "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
          done
 
 
@@ -61,19 +61,19 @@ jobs:
   #       cd crates/set
   #       for PYBIN in /opt/python/cp38*/bin; do
   #         "${PYBIN}/pip" install maturin
-  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
-  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
+  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
+  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
   #       done 
   #      cd ../cli
   #       for PYBIN in /opt/python/cp38*/bin; do
   #         "${PYBIN}/pip" install maturin
-  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
-  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
+  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility manylinux2014 --username "$MATURIN_USERNAME"
+  #         "${PYBIN}/maturin" publish -i "${PYBIN}/python" --no-sdist --skip-existing --compatibility musllinux_1_2 --username "$MATURIN_USERNAME"
   #       done
 
 
 
-  # Deploy for Windows and MoxOS 64 bits.
+  # Deploy for Windows 64 bits.
   # If Windows 32 bits neede, check e.g. https://github.com/marketplace/actions/setup-msys2
   build-windows-wheels:      
     runs-on: ${{ matrix.os }}
@@ -99,12 +99,11 @@ jobs:
        run: |
          pip install maturin
          cd crates/set
-         maturin publish --interpreter python${{matrix.python_version}} --skip-existing --username fxpineau 
+         maturin publish --interpreter python${{matrix.python_version}} --no-sdist --skip-existing --username fxpineau 
          cd ../cli
-         maturin publish --interpreter python${{matrix.python_version}} --skip-existing --username fxpineau 
+         maturin publish --interpreter python${{matrix.python_version}} --no-sdist --skip-existing --username fxpineau 
 
-  # Deploy for Windows and MoxOS 64 bits.
-  # If Windows 32 bits neede, check e.g. https://github.com/marketplace/actions/setup-msys2
+  # Deploy for MocOS 64 bits (also support M1 archi).
   build-macos-wheels:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -131,8 +130,8 @@ jobs:
          rustup target add aarch64-apple-darwin
          pip install maturin
          cd crates/set
-         maturin publish --interpreter python${{matrix.python_version}} --universal2 --skip-existing --username "$MATURIN_USERNAME"
+         maturin publish --interpreter python${{matrix.python_version}} --no-sdist --universal2 --skip-existing --username "$MATURIN_USERNAME"
          cd ../cli
-         maturin publish --interpreter python${{matrix.python_version}} --universal2 --skip-existing --username "$MATURIN_USERNAME"
+         maturin publish --interpreter python${{matrix.python_version}} --no-sdist --universal2 --skip-existing --username "$MATURIN_USERNAME"
 
 
diff --git a/.github/workflows/deploy_mac.yml b/.github/workflows/deploy_mac.yml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,17 @@
 # `moc` Change Log
 
+### x.x.x
+
+Released XXX-XX-XX
+
+### Added
+
+* Add `TTYPE1=RANGE` keyword in FITS files (TTYPE is optional in the FITS standard but without
+  it astropy seems not to be able to read the file)
+* Add the `CellHpxMOCIterator` trait to easily save S-MOC in FITS files compatible with v1.0
+  of the MOC standard.
+
+
 ## 0.9.0
 
 Released 2022-09-09
@@ -9,7 +21,6 @@ Released 2022-09-09
 * Print the deepest order in JSON output even when it contains no cell 
 
 
-
 ## 0.9.0-alpha
 
 Released 2022-06-17

diff --git a/crates/cli/CHANGELOG.md b/crates/cli/CHANGELOG.md
@@ -1,5 +1,13 @@
 # `moc-cli` Change Log
 
+## 0.5.3
+
+Realeased 2022-11-10
+
+* Add `TTYPE1=RANGE` keyword in FITS files (TTYPE is optional in the FITS standard but without
+  it astropy seems not to be able to read the file)
+* Add option `force_v1` to save a FITS file compatible with v1.0 of the MOC standard (S-MOC only).
+
 
 ## 0.5.2
 

diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "moc-cli"
-version = "0.5.2"
+version = "0.5.3"
 authors = ["F.-X. Pineau <[email protected]>"]
 description = "Command-line to create and manipulate HEALPix Multi-Order Coverages maps (MOCs)"
 license = "MIT OR Apache-2.0"

diff --git a/crates/cli/README.md b/crates/cli/README.md
@@ -1,5 +1,3 @@
-<meta charset="utf-8"/>
-
 # `moc-cli`
 
 A command-line to create and manipulate HEALPix **M**ulti-**O**rder **C**overage maps (**MOC**s),

diff --git a/crates/cli/src/from.rs b/crates/cli/src/from.rs
@@ -1408,6 +1408,7 @@ mod tests {
       separator: String::from(","),
       out: OutputFormat::Fits {
         force_u64: true,
+        force_v1: false,
         moc_id: None,
         moc_type: None,
         file: PathBuf::from("test/resources/xmmlog.range.stmoc.fits")

diff --git a/crates/cli/src/output.rs b/crates/cli/src/output.rs
@@ -1,28 +1,32 @@
 
-use std::fs::File;
-use std::io::{self, BufWriter};
-use std::str;
-use std::path::PathBuf;
-use std::error::Error;
+use std::{
+  str,
+  fs::File,
+  error::Error,
+  path::PathBuf,
+  io::{self, BufWriter},
+};
+
 use structopt::StructOpt;
 
-use moclib::idx::Idx;
-use moclib::qty::{MocQty, Hpx, Time, Frequency};
-use moclib::deser::fits;
-use moclib::moc::{
-  RangeMOCIterator, CellMOCIterator,
-  range::op::convert::{convert_to_u64, convert_from_u64}
-};
-use moclib::moc2d::{
-  RangeMOC2Iterator,
-  RangeMOC2ElemIt,
-  CellMOC2IntoIterator,
-  CellOrCellRangeMOC2IntoIterator,
-};
-use moclib::deser::{
-  fits::{ranges_to_fits_ivoa, ranges2d_to_fits_ivoa},
-  json::{to_json_aladin, cellmoc2d_to_json_aladin},
-  ascii::{to_ascii_ivoa, to_ascii_stream, moc2d_to_ascii_ivoa},
+use moclib::{
+  idx::Idx,
+  qty::{MocQty, Hpx, Time, Frequency},
+  moc::{
+    RangeMOCIterator, CellMOCIterator, CellHpxMOCIterator,
+    range::op::convert::{convert_to_u64, convert_from_u64}
+  },
+  moc2d::{
+    RangeMOC2Iterator,
+    RangeMOC2ElemIt,
+    CellMOC2IntoIterator,
+    CellOrCellRangeMOC2IntoIterator,
+  },
+  deser::{
+    fits::{self, ranges_to_fits_ivoa, ranges2d_to_fits_ivoa},
+    json::{to_json_aladin, cellmoc2d_to_json_aladin},
+    ascii::{to_ascii_ivoa, to_ascii_stream, moc2d_to_ascii_ivoa},
+  }
 };
 
 #[derive(StructOpt, Clone, Debug)]
@@ -54,6 +58,9 @@ pub enum OutputFormat {
     #[structopt(short = "-f", long = "--force-u64")]
     /// Force indices to be stored on u64 (ignored after operations involving 2 MOCs)
     force_u64: bool,
+    #[structopt(short = "-p", long = "--force-v1")]
+    /// Force compatibility with MOC v1.0 (i.e. save NUNIQ instead of Ranges; ignored if MOC is not a S-MOC)
+    force_v1: bool,
     #[structopt(short="-i", long = "--moc-id")]
     /// MOC ID to be written in the FITS header
     moc_id: Option<String>,
@@ -86,30 +93,42 @@ impl OutputFormat {
     matches!(self, OutputFormat::Fits { .. })
   }
 
+  pub fn is_fits_forced_to_v1_std(&self) -> bool {
+    matches!(self, OutputFormat::Fits { force_v1: true, .. })
+  }
+
   pub fn is_fits_forced_to_u64(&self) -> bool {
     matches!(self, OutputFormat::Fits { force_u64: true, .. })
   }
 
   pub fn is_fits_not_forced_to_u64(&self) -> bool {
     matches!(self, OutputFormat::Fits { force_u64: false, .. })
-
   }
 
   pub fn write_smoc_possibly_auto_converting_from_u64<I>(self, it: I) -> Result<(), Box<dyn Error>>
     where
       I: RangeMOCIterator<u64, Qty=Hpx<u64>>
   {
-    if self.is_fits_not_forced_to_u64() {
-      let depth = it.depth_max();
+    let depth = it.depth_max();
+    if self.is_fits_not_forced_to_u64() && depth <= Hpx::<u32>::MAX_DEPTH {
       if depth <= Hpx::<u16>::MAX_DEPTH {
-        self.write_moc(convert_from_u64::<Hpx<u64>, u16, Hpx<u16>, _>(it))
-      } else if depth <= Hpx::<u32>::MAX_DEPTH {
-        self.write_moc(convert_from_u64::<Hpx<u64>, u32, Hpx<u32>, _>(it))
+        if self.is_fits_forced_to_v1_std() {
+          self.write_smoc_fits_v1(convert_from_u64::<Hpx<u64>, u16, Hpx<u16>, _>(it))
+        } else {
+          self.write_moc(convert_from_u64::<Hpx<u64>, u16, Hpx<u16>, _>(it)) 
+        }
       } else {
-        self.write_moc(it)
+        assert!(depth <= Hpx::<u32>::MAX_DEPTH);
+        if self.is_fits_forced_to_v1_std() {
+          self.write_smoc_fits_v1(convert_from_u64::<Hpx<u64>, u32, Hpx<u32>, _>(it))
+        } else {
+          self.write_moc(convert_from_u64::<Hpx<u64>, u32, Hpx<u32>, _>(it))
+        }
       }
+    } else if self.is_fits_forced_to_v1_std() {
+      self.write_smoc_fits_v1(it)
     } else {
-      self.write_moc(it)
+      self.write_moc(it)  
     }
   }
 
@@ -118,7 +137,13 @@ impl OutputFormat {
       I: RangeMOCIterator<T, Qty=Hpx<T>>
   {
     if self.is_fits_forced_to_u64() {
-      self.write_moc(convert_to_u64::<T, Hpx<T>, _, Hpx<u64>>(it))
+      if self.is_fits_forced_to_v1_std() {
+        self.write_smoc_fits_v1(convert_to_u64::<T, Hpx<T>, _, Hpx<u64>>(it))
+      } else {
+        self.write_moc(convert_to_u64::<T, Hpx<T>, _, Hpx<u64>>(it)) 
+      }
+    } else if self.is_fits_forced_to_v1_std() {
+      self.write_smoc_fits_v1(it)
     } else {
       self.write_moc(it)
     }
@@ -129,9 +154,15 @@ impl OutputFormat {
       I: CellMOCIterator<T, Qty=Hpx<T>>
   {
     if self.is_fits_forced_to_u64() {
-      self.write_moc(convert_to_u64::<T, Hpx<T>, _, Hpx<u64>>(it.ranges()))
+      if self.is_fits_forced_to_v1_std() {
+        self.write_smoc_fits_v1_from_cells(it)
+      } else {
+        self.write_moc(convert_to_u64::<T, Hpx<T>, _, Hpx<u64>>(it.ranges()))
+      }
+    } else if self.is_fits_forced_to_v1_std() {
+        self.write_smoc_fits_v1_from_cells(it)
     } else {
-      self.write_moc_from_cells(it)
+        self.write_moc_from_cells(it)
     }
   }
 
@@ -216,7 +247,7 @@ impl OutputFormat {
         let file = File::create(path)?;
         to_json_aladin(it.cells(), &fold, "", BufWriter::new(file)).map_err(|e| e.into())
       },
-      OutputFormat::Fits { force_u64: _, moc_id, moc_type, file } => {
+      OutputFormat::Fits { force_u64: _, force_v1: _ , moc_id, moc_type, file } => {
         // Here I don't know how to convert the generic qty MocQty<T> into MocQty<u64>...
         let file = File::create(file)?;
         ranges_to_fits_ivoa(it, moc_id, moc_type, BufWriter::new(file)).map_err(|e| e.into())
@@ -228,6 +259,29 @@ impl OutputFormat {
     }
   }
 
+  pub fn write_smoc_fits_v1<T, I>(self, it: I) -> Result<(), Box<dyn Error>>
+    where
+      T: Idx,
+      I: RangeMOCIterator<T, Qty=Hpx<T>>
+  {
+    self.write_smoc_fits_v1_from_cells(it.cells())
+  }
+
+  pub fn write_smoc_fits_v1_from_cells<T, I>(self, it: I) -> Result<(), Box<dyn Error>>
+    where
+      T: Idx,
+      I: CellMOCIterator<T, Qty=Hpx<T>>
+  {
+    match self {
+      OutputFormat::Fits { force_u64: _, force_v1: _, moc_id, moc_type, file } => {
+        // Here I don't know how to convert the generic qty MocQty<T> into MocQty<u64>...
+        let file = File::create(file)?;
+        it.hpx_cells_to_fits_ivoa(moc_id, moc_type, BufWriter::new(file)).map_err(|e| e.into())
+      },
+      _ => unreachable!()
+    }
+  }
+
   pub fn write_moc_from_cells<T, Q, I>(self, it: I) -> Result<(), Box<dyn Error>>
     where
       T: Idx,
@@ -251,7 +305,7 @@ impl OutputFormat {
         let file = File::create(path)?;
         to_json_aladin(it, &fold, "", BufWriter::new(file)).map_err(|e| e.into())
       },
-      OutputFormat::Fits { force_u64: _, moc_id, moc_type, file } => {
+      OutputFormat::Fits { force_u64: _, force_v1: _, moc_id, moc_type, file } => {
         // Here I don't know how to convert the generic qty MocQty<T> into MocQty<u64>...
         let file = File::create(file)?;
         ranges_to_fits_ivoa(it.ranges(), moc_id, moc_type, BufWriter::new(file)).map_err(|e| e.into())
@@ -297,7 +351,7 @@ impl OutputFormat {
         let file = File::create(path)?;
         cellmoc2d_to_json_aladin(stmoc.into_cell_moc2_iter(), &fold, BufWriter::new(file)).map_err(|e| e.into())
       },
-      OutputFormat::Fits { force_u64: _, moc_id, moc_type, file } => {
+      OutputFormat::Fits { force_u64: _, force_v1: _ , moc_id, moc_type, file } => {
         // TODO handle the forced to u64??
         let file = File::create(file)?;
         ranges2d_to_fits_ivoa(stmoc, moc_id, moc_type, BufWriter::new(file)).map_err(|e| e.into())

diff --git a/crates/set/CHANGELOG.md b/crates/set/CHANGELOG.md
@@ -1,5 +1,14 @@
 # `moc-set` Change Log
 
+## 0.5.3
+
+Realeased 2022-11-10
+
+* Add `TTYPE1=RANGE` keyword in FITS files (TTYPE is optional in the FITS standard but without
+  it astropy seems not to be able to read the file)
+* Add option `force_v1` to extract a FITS file compatible with v1.0 of the MOC standard.
+
+
 ## 0.5.2
 
 Released 2022-09-12.

diff --git a/crates/set/Cargo.toml b/crates/set/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "moc-set"
-version = "0.5.2"
+version = "0.5.3"
 authors = ["F.-X. Pineau <[email protected]>"]
 description = "command-line tool to build, update and query a persistent set of HEALPix Multi-Order Coverages maps (MOCs)"
 license = "MIT OR Apache-2.0"