[SEDONA-708] Sedona should use PyArrow to get GeoPandas (#1794)

* Initial commit

* Separate the logic between Pandas and GeoPandas

* Work with GeoPandas < 1.0.0

* Refine the code structure

python/sedona/maps/SedonaMapUtils.py

@@ -18,6 +18,8 @@
 import json
 
 from sedona.sql.types import GeometryType
+from sedona.utils.geoarrow import dataframe_to_arrow
+from packaging.version import parse
 
 
 class SedonaMapUtils:
@@ -34,17 +36,25 @@ def __convert_to_gdf_or_pdf__(cls, df, rename=True, geometry_col=None):
         """
         if geometry_col is None:
             geometry_col = SedonaMapUtils.__get_geometry_col__(df)
-        pandas_df = df.toPandas()
+
+        # Convert the dataframe to arrow format, then to geopandas dataframe
+        # This is faster than converting directly to geopandas dataframe via toPandas
         if (
             geometry_col is None
         ):  # No geometry column found even after searching schema, return Pandas Dataframe
-            return pandas_df
+            data_pyarrow = dataframe_to_arrow(df)
+            return data_pyarrow.to_pandas()
         try:
             import geopandas as gpd
         except ImportError:
             msg = "GeoPandas is missing. You can install it manually or via apache-sedona[kepler-map] or apache-sedona[pydeck-map]."
             raise ImportError(msg) from None
-        geo_df = gpd.GeoDataFrame(pandas_df, geometry=geometry_col)
+        # From GeoPandas 1.0.0 onwards, the from_arrow method is available
+        if parse(gpd.__version__) >= parse("1.0.0"):
+            data_pyarrow = dataframe_to_arrow(df)
+            geo_df = gpd.GeoDataFrame.from_arrow(data_pyarrow)
+        else:
+            geo_df = gpd.GeoDataFrame(df.toPandas(), geometry=geometry_col)
         if geometry_col != "geometry" and rename is True:
             geo_df.rename_geometry("geometry", inplace=True)
         return geo_df

python/sedona/raster_utils/SedonaUtils.py

@@ -15,10 +15,13 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
+from sedona.maps.SedonaMapUtils import SedonaMapUtils
+
 
 class SedonaUtils:
     @classmethod
     def display_image(cls, df):
         from IPython.display import HTML, display
 
-        display(HTML(df.toPandas().to_html(escape=False)))
+        pdf = SedonaMapUtils.__convert_to_gdf_or_pdf__(df, rename=False)
+        display(HTML(pdf.to_html(escape=False)))