Skip to content

Commit

Permalink
Add pandas dependency for better CSV support
Browse files Browse the repository at this point in the history
  • Loading branch information
PierreRaybaut committed Apr 4, 2024
1 parent 5bb9c19 commit fb0989c
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 14 deletions.
28 changes: 23 additions & 5 deletions cdl/core/io/signal/funcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import re

import numpy as np
import pandas as pd


def read_csv(
Expand All @@ -30,12 +31,27 @@ def read_csv(
for delimiter, comments in (
(x, y) for x in (";", "\t", ",", " ") for y in ("#", None)
):
# Load everything readable (titles are eventually converted as NaNs)
try:
# Load everything readable (titles are eventually converted as NaNs)
xydata = np.genfromtxt(
filename, delimiter=delimiter, comments=comments, dtype=float
)
if np.all(np.isnan(xydata)):
# Headers are generally in the first 10 lines, so we try to skip the
# minimum number of lines before reading the data:
for skiprows in range(20):
try:
df = pd.read_csv(
filename,
dtype=float,
delimiter=delimiter,
comment=comments,
skiprows=skiprows,
)
xydata = df.to_numpy(float)
break
except (pd.errors.ParserError, ValueError):
pass
try:
if np.all(np.isnan(xydata)):
continue
except TypeError:
continue
# Removing columns with all but NaNs
xydata = xydata[:, ~np.all(np.isnan(xydata), axis=0)]
Expand Down Expand Up @@ -81,6 +97,8 @@ def read_csv(
break
except ValueError:
continue
if xydata is None:
raise ValueError("Unable to read CSV file")
return xydata, xlabel, xunit, ylabels, yunits, header


Expand Down
Loading

0 comments on commit fb0989c

Please sign in to comment.