diff --git a/src/json_builder.rs b/src/json_builder.rs new file mode 100644 index 0000000..76fee47 --- /dev/null +++ b/src/json_builder.rs @@ -0,0 +1,152 @@ +use std::{collections::HashMap, convert::TryFrom}; + +use lib0::any::Any; +use pyo3::{exceptions::PyTypeError, PyErr, PyObject, PyResult, Python}; + +use crate::shared_types::{CompatiblePyType, YPyType}; + +#[derive(Clone, Debug)] +pub(crate) struct JsonBuilder(String); + +impl JsonBuilder { + pub fn new() -> Self { + JsonBuilder(String::new()) + } + + pub fn append_json(&mut self, buildable: &T) -> Result<(), T::JsonError> { + let buffer = &mut self.0; + buildable.build_json(buffer) + } +} + +impl From for String { + fn from(json_builder: JsonBuilder) -> Self { + json_builder.0 + } +} + +pub(crate) trait JsonBuildable { + type JsonError; + fn build_json(&self, buffer: &mut String) -> Result<(), Self::JsonError>; +} + +impl<'a> JsonBuildable for CompatiblePyType<'a> { + type JsonError = PyErr; + + fn build_json(&self, buffer: &mut String) -> Result<(), Self::JsonError> { + match self { + CompatiblePyType::Bool(b) => { + let t: bool = b.extract().unwrap(); + buffer.push_str(if t { "true" } else { "false" }); + } + CompatiblePyType::Int(i) => buffer.push_str(&i.to_string()), + CompatiblePyType::Float(f) => buffer.push_str(&f.to_string()), + CompatiblePyType::String(s) => { + let string: String = s.extract().unwrap(); + buffer.reserve(string.len() + 2); + buffer.push_str("\""); + buffer.push_str(&string); + buffer.push_str("\""); + } + CompatiblePyType::List(list) => { + buffer.push_str("["); + let length = list.len(); + for (i, element) in list.iter().enumerate() { + CompatiblePyType::try_from(element)?.build_json(buffer)?; + if i + 1 < length { + buffer.push_str(","); + } + } + + buffer.push_str("]"); + } + CompatiblePyType::Dict(dict) => { + buffer.push_str("{"); + let length = dict.len(); + for (i, (k, v)) in dict.iter().enumerate() { + CompatiblePyType::try_from(k)?.build_json(buffer)?; + buffer.push_str(":"); + CompatiblePyType::try_from(v)?.build_json(buffer)?; + if i + 1 < length { + buffer.push_str(","); + } + } + buffer.push_str("}"); + } + CompatiblePyType::YType(y_type) => y_type.build_json(buffer)?, + CompatiblePyType::None => buffer.push_str("null"), + } + + Ok(()) + } +} + +impl<'a> JsonBuildable for YPyType<'a> { + type JsonError = PyErr; + + fn build_json(&self, buffer: &mut String) -> Result<(), Self::JsonError> { + let json = match self { + YPyType::Text(text) => Ok(text.borrow().to_json()), + YPyType::Array(array) => array.borrow().to_json(), + YPyType::Map(map) => map.borrow().to_json(), + xml => Err(PyTypeError::new_err(format!( + "XML elements cannot be converted to a JSON format: {xml}" + ))), + }; + buffer.push_str(&json?); + Ok(()) + } +} + +impl JsonBuildable for Any { + type JsonError = PyErr; + fn build_json(&self, buffer: &mut String) -> Result<(), Self::JsonError> { + self.to_json(buffer); + Ok(()) + } +} + +impl JsonBuildable for HashMap { + type JsonError = PyErr; + + fn build_json(&self, buffer: &mut String) -> Result<(), Self::JsonError> { + buffer.push_str("{"); + let res: PyResult<()> = Python::with_gil(|py| { + for (i, (k, py_obj)) in self.iter().enumerate() { + let value: CompatiblePyType = py_obj.extract(py)?; + if i != 0 { + buffer.push_str(","); + } + buffer.push_str(k); + buffer.push_str(":"); + value.build_json(buffer)?; + } + Ok(()) + }); + res?; + + buffer.push_str("}"); + Ok(()) + } +} + +impl JsonBuildable for Vec { + type JsonError = PyErr; + + fn build_json(&self, buffer: &mut String) -> Result<(), Self::JsonError> { + buffer.push_str("["); + let res: PyResult<()> = Python::with_gil(|py| { + self.iter().enumerate().try_for_each(|(i, object)| { + let py_type: CompatiblePyType = object.extract(py)?; + if i != 0 { + buffer.push_str(","); + } + py_type.build_json(buffer)?; + Ok(()) + }) + }); + res?; + buffer.push_str("]"); + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 296c8d4..9a65239 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ use pyo3::prelude::*; use pyo3::wrap_pyfunction; +mod json_builder; mod shared_types; mod type_conversions; mod y_array; diff --git a/src/shared_types.rs b/src/shared_types.rs index d660f78..6ec82fb 100644 --- a/src/shared_types.rs +++ b/src/shared_types.rs @@ -5,8 +5,9 @@ use crate::{ y_xml::{YXmlElement, YXmlText}, }; use pyo3::create_exception; +use pyo3::types as pytypes; use pyo3::{exceptions::PyException, prelude::*}; -use std::{convert::TryFrom, fmt::Display}; +use std::fmt::Display; use yrs::types::TYPE_REFS_XML_TEXT; use yrs::types::{TypeRefs, TYPE_REFS_ARRAY, TYPE_REFS_MAP, TYPE_REFS_TEXT}; use yrs::{types::TYPE_REFS_XML_ELEMENT, SubscriptionId}; @@ -50,6 +51,18 @@ pub enum SubId { Deep(DeepSubscription), } +#[derive(Clone)] +pub enum CompatiblePyType<'a> { + Bool(&'a pytypes::PyBool), + Int(&'a pytypes::PyInt), + Float(&'a pytypes::PyFloat), + String(&'a pytypes::PyString), + List(&'a pytypes::PyList), + Dict(&'a pytypes::PyDict), + YType(YPyType<'a>), + None, +} + #[derive(Clone)] pub enum SharedType { Integrated(I), @@ -67,68 +80,45 @@ impl SharedType { SharedType::Prelim(prelim) } } - -#[derive(FromPyObject)] -pub enum Shared { - Text(Py), - Array(Py), - Map(Py), - XmlElement(Py), - XmlText(Py), +#[derive(Clone)] +pub enum YPyType<'a> { + Text(&'a PyCell), + Array(&'a PyCell), + Map(&'a PyCell), + XmlElement(&'a PyCell), + XmlText(&'a PyCell), } -impl Shared { +impl<'a> YPyType<'a> { pub fn is_prelim(&self) -> bool { - Python::with_gil(|py| match self { - Shared::Text(v) => v.borrow(py).prelim(), - Shared::Array(v) => v.borrow(py).prelim(), - Shared::Map(v) => v.borrow(py).prelim(), - Shared::XmlElement(_) | Shared::XmlText(_) => false, - }) + match self { + YPyType::Text(v) => v.borrow().prelim(), + YPyType::Array(v) => v.borrow().prelim(), + YPyType::Map(v) => v.borrow().prelim(), + YPyType::XmlElement(_) | YPyType::XmlText(_) => false, + } } pub fn type_ref(&self) -> TypeRefs { match self { - Shared::Text(_) => TYPE_REFS_TEXT, - Shared::Array(_) => TYPE_REFS_ARRAY, - Shared::Map(_) => TYPE_REFS_MAP, - Shared::XmlElement(_) => TYPE_REFS_XML_ELEMENT, - Shared::XmlText(_) => TYPE_REFS_XML_TEXT, + YPyType::Text(_) => TYPE_REFS_TEXT, + YPyType::Array(_) => TYPE_REFS_ARRAY, + YPyType::Map(_) => TYPE_REFS_MAP, + YPyType::XmlElement(_) => TYPE_REFS_XML_ELEMENT, + YPyType::XmlText(_) => TYPE_REFS_XML_TEXT, } } } -impl Display for Shared { +impl<'a> Display for YPyType<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let info = Python::with_gil(|py| match self { - Shared::Text(t) => t.borrow(py).__str__(), - Shared::Array(a) => a.borrow(py).__str__(), - Shared::Map(m) => m.borrow(py).__str__(), - Shared::XmlElement(xml) => xml.borrow(py).__str__(), - Shared::XmlText(xml) => xml.borrow(py).__str__(), - }); + let info = match self { + YPyType::Text(t) => t.borrow().__str__(), + YPyType::Array(a) => a.borrow().__str__(), + YPyType::Map(m) => m.borrow().__str__(), + YPyType::XmlElement(xml) => xml.borrow().__str__(), + YPyType::XmlText(xml) => xml.borrow().__str__(), + }; write!(f, "{}", info) } } - -impl TryFrom for Shared { - type Error = PyErr; - - fn try_from(value: PyObject) -> Result { - Python::with_gil(|py| { - let value = value.as_ref(py); - - if let Ok(text) = value.extract() { - Ok(Shared::Text(text)) - } else if let Ok(array) = value.extract() { - Ok(Shared::Array(array)) - } else if let Ok(map) = value.extract() { - Ok(Shared::Map(map)) - } else { - Err(pyo3::exceptions::PyValueError::new_err( - "Could not extract Python value into a shared type.", - )) - } - }) - } -} diff --git a/src/type_conversions.rs b/src/type_conversions.rs index 12064c9..fb2ca21 100644 --- a/src/type_conversions.rs +++ b/src/type_conversions.rs @@ -14,7 +14,8 @@ use yrs::types::Events; use yrs::types::{Attrs, Branch, BranchPtr, Change, Delta, EntryChange, Value}; use yrs::{Array, Map, Text, Transaction}; -use crate::shared_types::{Shared, SharedType}; +use crate::shared_types::CompatiblePyType; +use crate::shared_types::{SharedType, YPyType}; use crate::y_array::YArray; use crate::y_array::YArrayEvent; use crate::y_map::YMap; @@ -31,7 +32,7 @@ pub trait ToPython { fn into_py(self, py: Python) -> PyObject; } -impl ToPython for Vec { +impl ToPython for Vec { fn into_py(self, py: Python) -> PyObject { let elements = self.into_iter().map(|v| v.into_py(py)); let arr: PyObject = pyo3::types::PyList::new(py, elements).into(); @@ -53,6 +54,41 @@ where } } +impl<'a> TryFrom<&'a PyAny> for CompatiblePyType<'a> { + type Error = PyErr; + + fn try_from(py_any: &'a PyAny) -> Result { + if let Ok(b) = py_any.downcast::() { + Ok(Self::Bool(b)) + } else if let Ok(i) = py_any.downcast::() { + Ok(Self::Int(i)) + } else if py_any.is_none() { + Ok(Self::None) + } else if let Ok(f) = py_any.downcast::() { + Ok(Self::Float(f)) + } else if let Ok(s) = py_any.downcast::() { + Ok(Self::String(s)) + } else if let Ok(list) = py_any.downcast::() { + Ok(Self::List(list)) + } else if let Ok(dict) = py_any.downcast::() { + Ok(Self::Dict(dict)) + } else if let Ok(v) = YPyType::try_from(py_any) { + Ok(Self::YType(v)) + } else { + Err(PyTypeError::new_err(format!( + "Cannot integrate this type into a YDoc: {py_any}" + ))) + } + } +} + +impl<'a> FromPyObject<'a> for CompatiblePyType<'a> { + fn extract(ob: &'a PyAny) -> PyResult { + Self::try_from(ob) + } +} + + impl ToPython for Delta { fn into_py(self, py: Python) -> PyObject { let result = pytypes::PyDict::new(py); @@ -147,27 +183,74 @@ impl<'a> IntoPy for EntryChangeWrapper<'a> { #[repr(transparent)] pub(crate) struct PyObjectWrapper(pub PyObject); +impl From for PyObjectWrapper { + fn from(value: PyObject) -> Self { + PyObjectWrapper(value) + } +} + +impl Deref for PyObjectWrapper { + type Target = PyObject; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + impl Prelim for PyObjectWrapper { + fn into_content(self, txn: &mut Transaction) -> (ItemContent, Option) { + Python::with_gil(|py| { + let valid_type: CompatiblePyType = self.0.extract(py).unwrap_or_else(|err| { + err.restore(py); + CompatiblePyType::None + }); + let (item_content, py_any) = valid_type.into_content(txn); + let wrapper: Option = py_any.map(|py_type| PyObjectWrapper(py_type.into())); + (item_content, wrapper) + }) + } + + fn integrate(self, txn: &mut Transaction, inner_ref: BranchPtr) { + Python::with_gil(|py| { + let valid_type: CompatiblePyType = self.0.extract(py).unwrap_or_else(|err| { + err.restore(py); + CompatiblePyType::None + }); + valid_type.integrate(txn, inner_ref); + }) + } +} + +impl<'a> From> for PyObject { + fn from(value: CompatiblePyType<'a>) -> Self { + match value { + CompatiblePyType::Bool(b) => b.into(), + CompatiblePyType::Int(i) => i.into(), + CompatiblePyType::Float(f) => f.into(), + CompatiblePyType::String(s) => s.into(), + CompatiblePyType::List(list) => list.into(), + CompatiblePyType::Dict(dict) => dict.into(), + CompatiblePyType::YType(y_type) => y_type.into(), + CompatiblePyType::None => Python::with_gil(|py| py.None()), + } + } +} + +impl<'a> Prelim for CompatiblePyType<'a> { fn into_content(self, _txn: &mut Transaction) -> (ItemContent, Option) { - let content = match PyObjectWrapper(self.0.clone()).try_into() { - Ok(any) => ItemContent::Any(vec![any]), - Err(err) => { - if Python::with_gil(|py| err.is_instance_of::(py)) { - let shared = Shared::try_from(self.0.clone()).unwrap(); - if shared.is_prelim() { - let branch = Branch::new(shared.type_ref(), None); - ItemContent::Type(branch) - } else { - Python::with_gil(|py| err.restore(py)); - ItemContent::Any(vec![]) - } - } else { - Python::with_gil(|py| err.restore(py)); - ItemContent::Any(vec![]) - } + let content = match self.clone() { + CompatiblePyType::YType(y_type) if y_type.is_prelim() => { + let branch = Branch::new(y_type.type_ref(), None); + Ok(ItemContent::Type(branch)) } + py_value => Any::try_from(py_value).map(|any| ItemContent::Any(vec![any])), }; + let content = content.unwrap_or_else(|err| { + Python::with_gil(|py| err.restore(py)); + ItemContent::Any(vec![]) + }); + let this = if let ItemContent::Type(_) = &content { Some(self) } else { @@ -178,97 +261,128 @@ impl Prelim for PyObjectWrapper { } fn integrate(self, txn: &mut Transaction, inner_ref: BranchPtr) { - if let Ok(shared) = Shared::try_from(self.0) { - if shared.is_prelim() { - Python::with_gil(|py| { - match shared { - Shared::Text(v) => { + match self { + CompatiblePyType::YType(y_type) if y_type.is_prelim() => { + match y_type { + YPyType::Text(v) => { let text = Text::from(inner_ref); - let mut y_text = v.borrow_mut(py); + let mut y_text = v.borrow_mut(); if let SharedType::Prelim(v) = y_text.0.to_owned() { text.push(txn, v.as_str()); } y_text.0 = SharedType::Integrated(text.clone()); } - Shared::Array(v) => { + YPyType::Array(v) => { let array = Array::from(inner_ref); - let mut y_array = v.borrow_mut(py); + let mut y_array = v.borrow_mut(); if let SharedType::Prelim(items) = y_array.0.to_owned() { let len = array.len(); YArray::insert_multiple_at(&array, txn, len, items); } y_array.0 = SharedType::Integrated(array.clone()); } - Shared::Map(v) => { + YPyType::Map(v) => { let map = Map::from(inner_ref); - let mut y_map = v.borrow_mut(py); - if let SharedType::Prelim(entries) = y_map.0.to_owned() { - for (k, v) in entries { - map.insert(txn, k, PyObjectWrapper(v)); + let mut y_map = v.borrow_mut(); + Python::with_gil(|py| { + if let SharedType::Prelim(ref entries) = y_map.0 { + for (k, v) in entries { + let x: CompatiblePyType = v.extract(py).unwrap_or_else(|err| { + err.restore(py); + CompatiblePyType::None + }); + map.insert(txn, k.to_owned(), x); + } } - } + }); + y_map.0 = SharedType::Integrated(map.clone()); } - Shared::XmlElement(_) | Shared::XmlText(_) => unreachable!("As defined in Shared::is_prelim(), neither XML type can ever exist outside a YDoc"), + YPyType::XmlElement(_) | YPyType::XmlText(_) => unreachable!("As defined in Shared::is_prelim(), neither XML type can ever exist outside a YDoc"), } - }) } + _ => () } } } -impl TryFrom for Any { +impl<'a> TryFrom> for Any { type Error = PyErr; - fn try_from(wrapper: PyObjectWrapper) -> Result { + fn try_from(py_type: CompatiblePyType<'a>) -> Result { const MAX_JS_NUMBER: i64 = 2_i64.pow(53) - 1; - let py_obj = wrapper.0; - Python::with_gil(|py| { - let v = py_obj.as_ref(py); - - if let Ok(b) = v.downcast::() { - Ok(Any::Bool(b.extract().unwrap())) - } else if let Ok(l) = v.downcast::() { - let num: i64 = l.extract().unwrap(); + match py_type { + CompatiblePyType::Bool(b) => Ok(Any::Bool(b.extract()?)), + CompatiblePyType::String(s) => Ok(Any::String(s.extract::()?.into_boxed_str())), + CompatiblePyType::Int(i) => { + let num: i64 = i.extract()?; if num > MAX_JS_NUMBER { Ok(Any::BigInt(num)) } else { Ok(Any::Number(num as f64)) } - } else if v.is_none() { - Ok(Any::Null) - } else if let Ok(f) = v.downcast::() { - Ok(Any::Number(f.extract().unwrap())) - } else if let Ok(s) = v.downcast::() { - let string: String = s.extract().unwrap(); - Ok(Any::String(string.into_boxed_str())) - } else if let Ok(list) = v.downcast::() { - let result: PyResult> = list + } + CompatiblePyType::Float(f) => Ok(Any::Number(f.extract()?)), + CompatiblePyType::List(l) => { + let result: PyResult> = l .into_iter() - .map(|py_val| PyObjectWrapper(py_val.into()).try_into()) + .map(|py_any|CompatiblePyType::try_from(py_any)?.try_into()) .collect(); result.map(|res| Any::Array(res.into_boxed_slice())) - } else if let Ok(dict) = v.downcast::() { - let result: PyResult> = dict + }, + CompatiblePyType::Dict(d) => { + let result: PyResult> = d .iter() .map(|(k, v)| { let key: String = k.extract()?; - let value = PyObjectWrapper(v.into()).try_into()?; + let value = CompatiblePyType::try_from(v)?.try_into()?; Ok((key, value)) }) .collect(); result.map(|res| Any::Map(Box::new(res))) - } else if let Ok(v) = Shared::try_from(PyObject::from(v)) { - Err(MultipleIntegrationError::new_err(format!( + }, + CompatiblePyType::None => Ok(Any::Null), + CompatiblePyType::YType(v) => Err(MultipleIntegrationError::new_err(format!( "Cannot integrate a nested Ypy object because is already integrated into a YDoc: {v}" - ))) - } else { - Err(PyTypeError::new_err(format!( - "Cannot integrate this type into a YDoc: {v}" - ))) - } - }) + ))), + } + } +} + +impl<'a> FromPyObject<'a> for YPyType<'a> { + fn extract(ob: &'a PyAny) -> PyResult { + Self::try_from(ob) + } +} + +impl<'a> From> for PyObject { + fn from(value: YPyType<'a>) -> Self { + match value { + YPyType::Text(text) => text.into(), + YPyType::Array(array) => array.into(), + YPyType::Map(map) => map.into(), + YPyType::XmlElement(xml) => xml.into(), + YPyType::XmlText(xml) => xml.into(), + } + } +} + +impl<'a> TryFrom<&'a PyAny> for YPyType<'a> { + type Error = PyErr; + + fn try_from(value: &'a PyAny) -> Result { + if let Ok(text) = value.extract() { + Ok(YPyType::Text(text)) + } else if let Ok(array) = value.extract() { + Ok(YPyType::Array(array)) + } else if let Ok(map) = value.extract() { + Ok(YPyType::Map(map)) + } else { + Err(pyo3::exceptions::PyValueError::new_err( + format!("Could not extract a Ypy type from this object: {value}") + )) + } } } diff --git a/src/y_array.rs b/src/y_array.rs index a8931c0..db24979 100644 --- a/src/y_array.rs +++ b/src/y_array.rs @@ -1,9 +1,9 @@ -use std::convert::TryInto; -use std::mem::ManuallyDrop; -use std::ops::DerefMut; +use std::convert::{TryFrom, TryInto}; +use crate::json_builder::JsonBuilder; use crate::shared_types::{ - DeepSubscription, DefaultPyErr, PreliminaryObservationException, ShallowSubscription, SubId, + CompatiblePyType, DeepSubscription, DefaultPyErr, PreliminaryObservationException, + ShallowSubscription, SubId, }; use crate::type_conversions::events_into_py; use crate::y_transaction::YTransaction; @@ -16,7 +16,7 @@ use pyo3::exceptions::PyIndexError; use crate::type_conversions::PyObjectWrapper; use pyo3::prelude::*; use pyo3::types::{PyList, PySlice, PySliceIndices}; -use yrs::types::array::{ArrayEvent, ArrayIter}; +use yrs::types::array::ArrayEvent; use yrs::types::DeepObservable; use yrs::{Array, SubscriptionId, Transaction}; @@ -83,7 +83,17 @@ impl YArray { } pub fn __str__(&self) -> String { - return self.to_json().to_string(); + match &self.0 { + SharedType::Integrated(y_array) => { + let any = y_array.to_json(); + let py_values = Python::with_gil(|py| any.into_py(py)); + py_values.to_string() + } + SharedType::Prelim(py_contents) => { + let py_values = Python::with_gil(|py| py_contents.clone().into_py(py)); + py_values.to_string() + } + } } pub fn __repr__(&self) -> String { @@ -91,14 +101,13 @@ impl YArray { } /// Converts an underlying contents of this `YArray` instance into their JSON representation. - pub fn to_json(&self) -> PyObject { - Python::with_gil(|py| match &self.0 { - SharedType::Integrated(v) => v.to_json().into_py(py), - SharedType::Prelim(v) => { - let py_ptrs: Vec = v.iter().cloned().collect(); - py_ptrs.into_py(py) - } - }) + pub fn to_json(&self) -> PyResult { + let mut json_builder = JsonBuilder::new(); + match &self.0 { + SharedType::Integrated(array) => json_builder.append_json(&array.to_json())?, + SharedType::Prelim(py_vec) => json_builder.append_json(py_vec)?, + } + Ok(json_builder.into()) } /// Adds a single item to the provided index in the array. pub fn insert(&mut self, txn: &mut YTransaction, index: u32, item: PyObject) -> PyResult<()> { @@ -147,10 +156,7 @@ impl YArray { /// Adds a single item to the end of the array pub fn append(&mut self, txn: &mut YTransaction, item: PyObject) { match &mut self.0 { - SharedType::Integrated(array) => { - let wrapper = PyObjectWrapper(item); - array.push_back(txn, wrapper) - } + SharedType::Integrated(array) => array.push_back(txn, PyObjectWrapper(item)), SharedType::Prelim(vec) => vec.push(item), } } @@ -183,12 +189,14 @@ impl YArray { SharedType::Integrated(v) => { v.move_to(txn, source, target); Ok(()) - }, - SharedType::Prelim(_) if source < 0 as u32 || target < 0 as u32 => Err(PyIndexError::default_message()), + } + SharedType::Prelim(_) if source < 0 as u32 || target < 0 as u32 => { + Err(PyIndexError::default_message()) + } SharedType::Prelim(v) if source < v.len() as u32 && target < v.len() as u32 => { if source < target { let el = v.remove(source as usize); - v.insert((target-1) as usize, el); + v.insert((target - 1) as usize, el); } else if source > target { let el = v.remove(source as usize); v.insert(target as usize, el); @@ -227,15 +235,21 @@ impl YArray { SharedType::Integrated(v) => { v.move_range_to(txn, start, true, end, false, target); Ok(()) - }, + } // y-rs does nothing if end < start // SharedType::Prelim(_) if end < start => Err(PyIndexError::default_message()), - SharedType::Prelim(_) if start < 0 as u32 || end < 0 as u32 || target < 0 as u32 => Err(PyIndexError::default_message()), - SharedType::Prelim(v) if start > v.len() as u32 || end > v.len() as u32 || target > v.len() as u32 => Err(PyIndexError::default_message()), + SharedType::Prelim(_) if start < 0 as u32 || end < 0 as u32 || target < 0 as u32 => { + Err(PyIndexError::default_message()) + } + SharedType::Prelim(v) + if start > v.len() as u32 || end > v.len() as u32 || target > v.len() as u32 => + { + Err(PyIndexError::default_message()) + } // It doesn't make sense to move a range into the same range (it's basically a no-op). - SharedType::Prelim(_) if target >= start && target <= end => Ok(()), + SharedType::Prelim(_) if target >= start && target <= end => Ok(()), SharedType::Prelim(v) => { let mut i: usize = 0; @@ -276,22 +290,19 @@ impl YArray { /// # document on machine A /// doc = YDoc() /// array = doc.get_array('name') - /// for item in array.values()): - /// print(item) + /// for item in array: + /// print(item) /// /// ``` - pub fn __iter__(&self) -> YArrayIterator { - let inner_iter = match &self.0 { - SharedType::Integrated(v) => unsafe { - let this: *const Array = v; - InnerYArrayIter::Integrated((*this).iter()) - }, - SharedType::Prelim(v) => unsafe { - let this: *const Vec = v; - InnerYArrayIter::Prelim((*this).iter()) - }, - }; - YArrayIterator(ManuallyDrop::new(inner_iter)) + pub fn __iter__(&self) -> PyObject { + Python::with_gil(|py| { + let list: PyObject = match &self.0 { + SharedType::Integrated(arr) => arr.to_json().into_py(py), + SharedType::Prelim(arr) => arr.clone().into_py(py), + }; + let any = list.as_ref(py); + any.iter().unwrap().into_py(py) + }) } /// Subscribes to all operations happening over this instance of `YArray`. All changes are @@ -429,28 +440,33 @@ impl YArray { pub fn insert_multiple_at(dst: &Array, txn: &mut Transaction, index: u32, src: Vec) { let mut j = index; let mut i = 0; - while i < src.len() { - let mut anys: Vec = Vec::default(); + Python::with_gil(|py| { while i < src.len() { - if let Ok(any) = PyObjectWrapper(src[i].clone()).try_into() { - anys.push(any); - i += 1; - } else { - break; + let mut anys: Vec = Vec::default(); + while i < src.len() { + let converted_item: PyResult = + CompatiblePyType::try_from(src[i].as_ref(py)).and_then(Any::try_from); + if let Ok(any) = converted_item { + anys.push(any); + i += 1; + } else { + println!("{converted_item:?}"); + break; + } } - } - if !anys.is_empty() { - let len = anys.len() as u32; - dst.insert_range(txn, j, anys); - j += len; - } else { - let wrapper = PyObjectWrapper(src[i].clone()); - dst.insert(txn, j, wrapper); - i += 1; - j += 1; + if !anys.is_empty() { + let len = anys.len() as u32; + dst.insert_range(txn, j, anys); + j += len; + } else { + let wrapper = PyObjectWrapper(src[i].clone()); + dst.insert(txn, j, wrapper); + i += 1; + j += 1; + } } - } + }) } fn py_iter(iterable: PyObject) -> PyResult> { @@ -474,44 +490,6 @@ pub enum Index<'a> { Slice(&'a PySlice), } -enum InnerYArrayIter { - Integrated(ArrayIter<'static>), - Prelim(std::slice::Iter<'static, PyObject>), -} - -#[pyclass(unsendable)] -pub struct YArrayIterator(ManuallyDrop); - -impl Drop for YArrayIterator { - fn drop(&mut self) { - unsafe { ManuallyDrop::drop(&mut self.0) } - } -} - -impl Iterator for YArrayIterator { - type Item = PyObject; - - fn next(&mut self) -> Option { - match self.0.deref_mut() { - InnerYArrayIter::Integrated(iter) => { - Python::with_gil(|py| iter.next().map(|v| v.into_py(py))) - } - InnerYArrayIter::Prelim(iter) => iter.next().cloned(), - } - } -} - -#[pymethods] -impl YArrayIterator { - pub fn __iter__(slf: PyRef) -> PyRef { - slf - } - - pub fn __next__(mut slf: PyRefMut) -> Option { - slf.next() - } -} - /// Event generated by `YArray.observe` method. Emitted during transaction commit phase. #[pyclass(unsendable)] pub struct YArrayEvent { diff --git a/src/y_map.rs b/src/y_map.rs index 71507c1..a6ac079 100644 --- a/src/y_map.rs +++ b/src/y_map.rs @@ -2,12 +2,14 @@ use pyo3::exceptions::{PyKeyError, PyTypeError}; use pyo3::prelude::*; use pyo3::types::PyDict; use std::collections::HashMap; + use std::mem::ManuallyDrop; use std::ops::DerefMut; use yrs::types::map::{MapEvent, MapIter}; use yrs::types::DeepObservable; use yrs::{Map, SubscriptionId, Transaction}; +use crate::json_builder::JsonBuilder; use crate::shared_types::{ DeepSubscription, DefaultPyErr, PreliminaryObservationException, ShallowSubscription, SharedType, SubId, @@ -73,19 +75,13 @@ impl YMap { } pub fn __str__(&self) -> String { - return self.to_json().unwrap().to_string(); + Python::with_gil(|py| match &self.0 { + SharedType::Integrated(y_map) => y_map.to_json().into_py(py).to_string(), + SharedType::Prelim(contents) => contents.clone().into_py(py).to_string(), + }) } pub fn __dict__(&self) -> PyResult { - self.to_json() - } - - pub fn __repr__(&self) -> String { - format!("YMap({})", self.__str__()) - } - - /// Converts contents of this `YMap` instance into a JSON representation. - pub fn to_json(&self) -> PyResult { Python::with_gil(|py| match &self.0 { SharedType::Integrated(v) => Ok(v.to_json().into_py(py)), SharedType::Prelim(v) => { @@ -98,6 +94,20 @@ impl YMap { }) } + pub fn __repr__(&self) -> String { + format!("YMap({})", self.__str__()) + } + + /// Converts contents of this `YMap` instance into a JSON representation. + pub fn to_json(&self) -> PyResult { + let mut json_builder = JsonBuilder::new(); + match &self.0 { + SharedType::Integrated(dict) => json_builder.append_json(&dict.to_json())?, + SharedType::Prelim(dict) => json_builder.append_json(dict)?, + } + Ok(json_builder.into()) + } + /// Sets a given `key`-`value` entry within this instance of `YMap`. If another entry was /// already stored under given `key`, it will be overridden with new `value`. pub fn set(&mut self, txn: &mut YTransaction, key: &str, value: PyObject) { diff --git a/src/y_text.rs b/src/y_text.rs index d8d9a00..258b0ed 100644 --- a/src/y_text.rs +++ b/src/y_text.rs @@ -1,8 +1,8 @@ use crate::shared_types::{ - DeepSubscription, DefaultPyErr, IntegratedOperationException, PreliminaryObservationException, - ShallowSubscription, SharedType, SubId, + CompatiblePyType, DeepSubscription, DefaultPyErr, IntegratedOperationException, + PreliminaryObservationException, ShallowSubscription, SharedType, SubId, }; -use crate::type_conversions::{events_into_py, PyObjectWrapper, ToPython}; +use crate::type_conversions::{events_into_py, ToPython}; use crate::y_transaction::YTransaction; use lib0::any::Any; use pyo3::prelude::*; @@ -86,9 +86,7 @@ impl YText { /// Returns an underlying shared string stored in this data type. pub fn to_json(&self) -> String { - let mut json_string = String::new(); - Any::String(self.__str__().into_boxed_str()).to_json(&mut json_string); - json_string + format!("\"{}\"", self.__str__()) } /// Inserts a given `chunk` of text into this `YText` instance, starting at a given `index`. @@ -136,11 +134,14 @@ impl YText { ) -> PyResult<()> { match &mut self.0 { SharedType::Integrated(text) => { - let content = PyObjectWrapper(embed).try_into()?; + let content: PyResult = Python::with_gil(|py| { + let py_type: CompatiblePyType = embed.extract(py)?; + py_type.try_into() + }); if let Some(Ok(attrs)) = attributes.map(Self::parse_attrs) { - text.insert_embed_with_attributes(txn, index, content, attrs) + text.insert_embed_with_attributes(txn, index, content?, attrs) } else { - text.insert_embed(txn, index, content) + text.insert_embed(txn, index, content?) } Ok(()) } @@ -246,14 +247,16 @@ impl YText { impl YText { fn parse_attrs(attrs: HashMap) -> PyResult { - attrs - .into_iter() - .map(|(k, v)| { - let key = Rc::from(k); - let value = PyObjectWrapper(v).try_into()?; - Ok((key, value)) - }) - .collect() + Python::with_gil(|py| { + attrs + .into_iter() + .map(|(k, v)| { + let key = Rc::from(k); + let value: CompatiblePyType = v.extract(py)?; + Ok((key, value.try_into()?)) + }) + .collect() + }) } } diff --git a/tests/test_y_array.py b/tests/test_y_array.py index 5e58e7f..eae5066 100644 --- a/tests/test_y_array.py +++ b/tests/test_y_array.py @@ -2,7 +2,8 @@ import pytest from y_py import YDoc, YArray, YArrayEvent - +from copy import deepcopy +import json def test_inserts(): d1 = YDoc(1) @@ -16,16 +17,14 @@ def test_inserts(): expected = [1, 2.5, "hello", ["world"], True, {"key": "value"}] - value = x.to_json() - assert value == expected + assert list(x) == expected d2 = YDoc(2) x = d2.get_array("test") exchange_updates([d1, d2]) - value = x.to_json() - assert value == expected + assert list(x) == expected def test_to_string(): @@ -34,28 +33,39 @@ def test_to_string(): assert str(arr) == expected_str assert arr.__repr__() == f"YArray({expected_str})" +def test_to_json(): + contents = [7, "awesome", True, ["nested"], {"testing": "dicts"}] + doc = YDoc() + prelim = YArray(deepcopy(contents)) + integrated = doc.get_array("arr") + with doc.begin_transaction() as txn: + integrated.extend(txn, contents) + expected_json = '[7,"awesome",true,["nested"],{"testing":"dicts"}]' + assert integrated.to_json() == expected_json + assert prelim.to_json() == expected_json + + # ensure that it works with python json + assert json.loads(integrated.to_json()) == contents def test_inserts_nested(): d1 = YDoc() x = d1.get_array("test") - + to_list = lambda arr : [list(x) if type(x) == YArray else x for x in arr] nested = YArray() d1.transact(lambda txn: nested.append(txn, "world")) d1.transact(lambda txn: x.insert_range(txn, 0, [1, 2, nested, 3, 4])) d1.transact(lambda txn: nested.insert(txn, 0, "hello")) expected = [1, 2, ["hello", "world"], 3, 4] - - value = d1.transact(lambda txn: x.to_json()) - assert value == expected + assert type(x[2]) == YArray + assert to_list(x) == expected d2 = YDoc() x = d2.get_array("test") exchange_updates([d1, d2]) - value = x.to_json() - assert value == expected + assert to_list(x) == expected def test_delete(): @@ -74,13 +84,12 @@ def test_delete(): expected = [1, "I'm here too!", True] - value = x.to_json() - assert value == expected + assert list(x) == expected with d1.begin_transaction() as txn: x.delete(txn, 1) - assert x.to_json() == [1.0, True] + assert list(x) == [1.0, True] with pytest.raises(IndexError): with d1.begin_transaction() as txn: x.delete(txn, 2) @@ -90,8 +99,7 @@ def test_delete(): exchange_updates([d1, d2]) - value = x.to_json() - assert value == [1.0, True] + assert list(x) == [1.0, True] def test_get(): @@ -179,7 +187,7 @@ def callback(e: YArrayEvent): # insert initial data to an empty YArray with d1.begin_transaction() as txn: x.insert_range(txn, 0, [1, 2, 3, 4]) - assert target.to_json() == x.to_json() + assert list(target) == list(x) assert delta == [{"insert": [1, 2, 3, 4]}] target = None @@ -188,7 +196,7 @@ def callback(e: YArrayEvent): # remove 2 items from the middle with d1.begin_transaction() as txn: x.delete_range(txn, 1, 2) - assert target.to_json() == x.to_json() + assert list(target) == list(x) assert delta == [{"retain": 1}, {"delete": 2}] target = None @@ -197,7 +205,7 @@ def callback(e: YArrayEvent): # insert item in the middle with d1.begin_transaction() as txn: x.insert(txn, 1, 5) - assert target.to_json() == x.to_json() + assert list(target) == list(x) assert delta == [{"retain": 1}, {"insert": [5]}] target = None @@ -255,21 +263,21 @@ def test_move_to(): arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_to(t, 0, 10)) - assert arr.to_json() == [1,2,3,4,5,6,7,8,9,0] + assert list(arr) == [1,2,3,4,5,6,7,8,9,0] # Move 9 to 0 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_to(t, 9, 0)) - assert arr.to_json() == [9,0,1,2,3,4,5,6,7,8] + assert list(arr) == [9,0,1,2,3,4,5,6,7,8] # Move 6 to 5 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_to(t, 6, 5)) - assert arr.to_json() == [0,1,2,3,4,6,5,7,8,9] + assert list(arr) == [0,1,2,3,4,6,5,7,8,9] # Move -1 to 5 with doc.begin_transaction() as t: @@ -297,21 +305,21 @@ def test_move_range_to(): arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3]) doc.transact(lambda t: arr.move_range_to(t, 1, 2, 4)) - assert arr.to_json() == [0,3,1,2] + assert list(arr) == [0,3,1,2] # Move 0-0 to 10 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 0, 0, 10)) - assert arr.to_json() == [1,2,3,4,5,6,7,8,9,0] + assert list(arr) == [1,2,3,4,5,6,7,8,9,0] # Move 0-1 to 10 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 0, 1, 10)) - assert arr.to_json() == [2,3,4,5,6,7,8,9,0,1] + assert list(arr) == [2,3,4,5,6,7,8,9,0,1] # Move 3-5 to 7 @@ -319,49 +327,49 @@ def test_move_range_to(): arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 3, 5, 7)) - assert arr.to_json() == [0,1,2,6,3,4,5,7,8,9] + assert list(arr) == [0,1,2,6,3,4,5,7,8,9] # Move 1-0 to 10 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 1, 0, 10)) - assert arr.to_json() == [0,1,2,3,4,5,6,7,8,9] + assert list(arr) == [0,1,2,3,4,5,6,7,8,9] # Move 3-5 to 5 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 3, 5, 5)) - assert arr.to_json() == [0,1,2,3,4,5,6,7,8,9] + assert list(arr) == [0,1,2,3,4,5,6,7,8,9] # Move 9-9 to 0 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 9, 9, 0)) - assert arr.to_json() == [9,0,1,2,3,4,5,6,7,8] + assert list(arr) == [9,0,1,2,3,4,5,6,7,8] # Move 8-9 to 0 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 8, 9, 0)) - assert arr.to_json() == [8,9,0,1,2,3,4,5,6,7] + assert list(arr) == [8,9,0,1,2,3,4,5,6,7] # Move 4-6 to 3 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 4, 6, 3)) - assert arr.to_json() == [0,1,2,4,5,6,3,7,8,9] + assert list(arr) == [0,1,2,4,5,6,3,7,8,9] # Move 3-5 to 3 with doc.begin_transaction() as t: arr.delete_range(t, 0, len(arr)) arr.extend(t, [0,1,2,3,4,5,6,7,8,9]) doc.transact(lambda t: arr.move_range_to(t, 3, 5, 3)) - assert arr.to_json() == [0,1,2,3,4,5,6,7,8,9] + assert list(arr) == [0,1,2,3,4,5,6,7,8,9] # Move -1-2 to 5 with doc.begin_transaction() as t: diff --git a/tests/test_y_doc.py b/tests/test_y_doc.py index 2962695..4127508 100644 --- a/tests/test_y_doc.py +++ b/tests/test_y_doc.py @@ -41,7 +41,7 @@ def test_encoding(): state_vec = Y.encode_state_vector(receiver) update = Y.encode_state_as_update(doc, state_vec) Y.apply_update(receiver, update) - value = receiver.get_array("test").to_json() + value = list(receiver.get_array("test")) assert value == contents @@ -59,7 +59,7 @@ def test_boolean_encoding(): state_vec = Y.encode_state_vector(receiver) update = Y.encode_state_as_update(doc, state_vec) Y.apply_update(receiver, update) - value = receiver.get_array("test").to_json() + value = list(receiver.get_array("test")) assert type(value[0]) == type(True) diff --git a/tests/test_y_map.py b/tests/test_y_map.py index 5b18d06..e610d8c 100644 --- a/tests/test_y_map.py +++ b/tests/test_y_map.py @@ -1,3 +1,5 @@ +from copy import deepcopy +import json import pytest import y_py as Y from y_py import YMap, YMapEvent @@ -41,6 +43,20 @@ def test_set(): assert value == "value2" +def test_to_json(): + contents = {"emojis": [ + {"icon":"👍", "description": "thumbs up", "positive":True}, + {"icon":"👎", "description": "thumbs down", "positive":False}, + ]} + doc = Y.YDoc() + prelim = Y.YMap(deepcopy(contents)) + integrated = doc.get_map("map") + with doc.begin_transaction() as txn: + integrated.update(txn, contents) + # ensure that it works with python json + assert json.loads(integrated.to_json()) == contents + + def test_update(): doc = Y.YDoc() ymap = doc.get_map("dict") @@ -50,13 +66,13 @@ def test_update(): # Test updating with a dictionary with doc.begin_transaction() as txn: ymap.update(txn, dict_vals) - assert ymap.to_json() == dict_vals + assert dict(ymap) == dict_vals # Test updating with an iterator ymap = doc.get_map("tuples") with doc.begin_transaction() as txn: ymap.update(txn, tuple_vals) - assert ymap.to_json() == dict_vals + assert dict(ymap) == dict_vals # Test non-string key error with pytest.raises(Exception) as e: @@ -74,13 +90,11 @@ def test_set_nested(): x = d1.get_map("test") nested = Y.YMap({"a": "A"}) - # check out to_json(), setting a nested map in set(), adding to an integrated value - d1.transact(lambda txn: x.set(txn, "key", nested)) d1.transact(lambda txn: nested.set(txn, "b", "B")) - json = x.to_json() - assert json == {"key": {"a": "A", "b": "B"}} + assert type(x["key"]) == Y.YMap + assert {k : dict(v) for k, v in x.items()} == {"key": {"a": "A", "b": "B"}} def test_pop(): @@ -179,9 +193,6 @@ def test_observer(): target = None entries = None - def get_value(x): - return x.to_json() - def callback(e: YMapEvent): nonlocal target nonlocal entries @@ -195,7 +206,7 @@ def callback(e: YMapEvent): x.set(txn, "key1", "value1") x.set(txn, "key2", 2) - assert get_value(target) == get_value(x) + assert dict(target) == dict(x) assert entries == { "key1": {"action": "add", "newValue": "value1"}, "key2": {"action": "add", "newValue": 2}, @@ -209,7 +220,7 @@ def callback(e: YMapEvent): x.pop(txn, "key1") x.set(txn, "key2", "value2") - assert get_value(target) == get_value(x) + assert dict(target) == dict(x) assert entries == { "key1": {"action": "delete", "oldValue": "value1"}, "key2": {"action": "update", "oldValue": 2, "newValue": "value2"}, diff --git a/y_py.pyi b/y_py.pyi index 17371d4..05199e1 100644 --- a/y_py.pyi +++ b/y_py.pyi @@ -552,7 +552,7 @@ class YArray: Returns: The string representation of YArray wrapped in `YArray()` """ - def to_json(self) -> List[Any]: + def to_json(self) -> str: """ Converts an underlying contents of this `YArray` instance into their JSON representation. """ @@ -725,12 +725,19 @@ class YMap: Returns: The string representation of the `YMap`. """ + + def __dict__(self) -> dict: + """ + Returns: + Contents of the `YMap` inside a Python dictionary. + """ + def __repr__(self) -> str: """ Returns: The string representation of the `YMap` wrapped in 'YMap()' """ - def to_json(self) -> dict: + def to_json(self) -> str: """ Converts contents of this `YMap` instance into a JSON representation. """