The following are code examples for showing how to use . They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don’t like. You can also save this page to your account.
Example 1
def _get_data_dims(self, input_fname):
"""Briefly scan the data file for info"""
# raw data formatting is nsamps by nchans + counter
data = np.genfromtxt(input_fname, delimiter=',', comments='%',
skip_footer=1)
diff = np.abs(np.diff(data[:, 0]))
diff = np.mod(diff, 254) - 1
missing_idx = np.where(diff != 0)[0]
missing_samps = diff[missing_idx].astype(int)
nsamps, nchan = data.shape
# add the missing samples
nsamps += sum(missing_samps)
# remove the tracker column
nchan -= 1
del data
return nsamps, nchan Example 2
def get_adjacency_matrix(out_dir, sid, expt_id):
"Returns the adjacency matrix"
vec_path = pjoin(out_dir, sid, '{}_graynet.csv'.format(expt_id))
edge_vec = np.genfromtxt(vec_path)
matrix_size = np.int64( (1.0 + np.sqrt(1.0+8.0*len(edge_vec)))/2.0 )
edge_mat = np.zeros([matrix_size, matrix_size])
# making this symmetric as required by nilearn's plot_connectome (stupid)
# upper tri; diag +1; # lower tri; diag -1
upper_tri = np.triu_indices_from(edge_mat, +1)
lower_tri = np.tril_indices_from(edge_mat, -1)
edge_mat[upper_tri] = edge_vec
edge_mat[lower_tri] = edge_mat.T[lower_tri]
return edge_mat Example 3
def _block2df(block,obstypes,svnames,svnum):
"""
input: block of text corresponding to one time increment INTERVAL of RINEX file
output: 2-D array of float64 data from block. Future: consider whether best to use Numpy, Pandas, or Xray.
"""
nobs = len(obstypes)
stride=3
strio = BytesIO(block.encode())
barr = np.genfromtxt(strio, delimiter=(14,1,1)*5).reshape((svnum,-1), order='C')
data = barr[:,0:nobs*stride:stride]
lli = barr[:,1:nobs*stride:stride]
ssi = barr[:,2:nobs*stride:stride]
data = np.vstack(([data.T],[lli.T],[ssi.T])).T
return data Example 4
def _block2df(block,obstypes,svnames,svnum):
"""
input: block of text corresponding to one time increment INTERVAL of RINEX file
output: 2-D array of float64 data from block. Future: consider whether best to use Numpy, Pandas, or Xray.
"""
nobs = len(obstypes)
stride=3
strio = BytesIO(block.encode())
barr = np.genfromtxt(strio, delimiter=(14,1,1)*5).reshape((svnum,-1), order='C')
data = barr[:,0:nobs*stride:stride]
lli = barr[:,1:nobs*stride:stride]
ssi = barr[:,2:nobs*stride:stride]
data = np.vstack(([data.T],[lli.T],[ssi.T])).T
return data Example 5
def _block2df(block,obstypes,svnames,svnum):
"""
input: block of text corresponding to one time increment INTERVAL of RINEX file
output: 2-D array of float64 data from block.
"""
nobs = len(obstypes)
stride=3
strio = BytesIO(block.encode())
barr = np.genfromtxt(strio, delimiter=(14,1,1)*5).reshape((svnum,-1), order='C')
data = barr[:,0:nobs*stride:stride]
lli = barr[:,1:nobs*stride:stride]
ssi = barr[:,2:nobs*stride:stride]
data = np.vstack(([data],[lli],[ssi])).T #4D numpy array
return data Example 6
def _read_horizons_file(self):
"""
reads standard output from JPL Horizons into self.data_lists
"""
# Read in the file
self._get_start_end()
data = np.genfromtxt(
self.file_properties['file_name'],
dtype=[('date', 'S17'), ('ra_dec', 'S23'), ('distance', 'f8'),
('foo', 'S23')],
delimiter=[18, 29, 18, 24], autostrip=True,
skip_header=self.file_properties['start_ind'] + 1,
skip_footer=(self.file_properties['line_count'] -
self.file_properties['stop_ind']))
# Fix time format
for (i, date) in enumerate(data['date']):
data['date'][i] = Utils.date_change(date)
self.data_lists = data Example 7
def test_skip_footer_with_invalid(self):
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
basestr = '1 1\n2 2\n3 3\n4 4\n5 \n6 \n7 \n'
# Footer too small to get rid of all invalid values
assert_raises(ValueError, np.genfromtxt,
TextIO(basestr), skip_footer=1)
# except ValueError:
# pass
a = np.genfromtxt(
TextIO(basestr), skip_footer=1, invalid_raise=False)
assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
#
a = np.genfromtxt(TextIO(basestr), skip_footer=3)
assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
#
basestr = '1 1\n2 \n3 3\n4 4\n5 \n6 6\n7 7\n'
a = np.genfromtxt(
TextIO(basestr), skip_footer=1, invalid_raise=False)
assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.], [6., 6.]]))
a = np.genfromtxt(
TextIO(basestr), skip_footer=3, invalid_raise=False)
assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.]])) Example 8
def test_commented_header(self):
# Check that names can be retrieved even if the line is commented out.
data = TextIO("""
#gender age weight
M 21 72.100000
F 35 58.330000
M 33 21.99
""")
# The # is part of the first name and should be deleted automatically.
test = np.genfromtxt(data, names=True, dtype=None)
ctrl = np.array([('M', 21, 72.1), ('F', 35, 58.33), ('M', 33, 21.99)],
dtype=[('gender', '|S1'), ('age', int), ('weight', float)])
assert_equal(test, ctrl)
# Ditto, but we should get rid of the first element
data = TextIO(b"""
# gender age weight
M 21 72.100000
F 35 58.330000
M 33 21.99
""")
test = np.genfromtxt(data, names=True, dtype=None)
assert_equal(test, ctrl) Example 9
def test_dtype_with_object(self):
# Test using an explicit dtype with an object
data = """ 1; 2001-01-01
2; 2002-01-31 """
ndtype = [('idx', int), ('code', np.object)]
func = lambda s: strptime(s.strip(), "%Y-%m-%d")
converters = {1: func}
test = np.genfromtxt(TextIO(data), delimiter=";", dtype=ndtype,
converters=converters)
control = np.array(
[(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
dtype=ndtype)
assert_equal(test, control)
ndtype = [('nest', [('idx', int), ('code', np.object)])]
try:
test = np.genfromtxt(TextIO(data), delimiter=";",
dtype=ndtype, converters=converters)
except NotImplementedError:
pass
else:
errmsg = "Nested dtype involving objects should be supported."
raise AssertionError(errmsg) Example 10
def test_replace_space(self):
# Test the 'replace_space' option
txt = "A.A, B (B), C:C\n1, 2, 3.14"
# Test default: replace ' ' by '_' and delete non-alphanum chars
test = np.genfromtxt(TextIO(txt),
delimiter=",", names=True, dtype=None)
ctrl_dtype = [("AA", int), ("B_B", int), ("CC", float)]
ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
assert_equal(test, ctrl)
# Test: no replace, no delete
test = np.genfromtxt(TextIO(txt),
delimiter=",", names=True, dtype=None,
replace_space='', deletechars='')
ctrl_dtype = [("A.A", int), ("B (B)", int), ("C:C", float)]
ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
assert_equal(test, ctrl)
# Test: no delete (spaces are replaced by _)
test = np.genfromtxt(TextIO(txt),
delimiter=",", names=True, dtype=None,
deletechars='')
ctrl_dtype = [("A.A", int), ("B_(B)", int), ("C:C", float)]
ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
assert_equal(test, ctrl) Example 11
def test_names_with_usecols_bug1636(self):
# Make sure we pick up the right names w/ usecols
data = "A,B,C,D,E\n0,1,2,3,4\n0,1,2,3,4\n0,1,2,3,4"
ctrl_names = ("A", "C", "E")
test = np.genfromtxt(TextIO(data),
dtype=(int, int, int), delimiter=",",
usecols=(0, 2, 4), names=True)
assert_equal(test.dtype.names, ctrl_names)
#
test = np.genfromtxt(TextIO(data),
dtype=(int, int, int), delimiter=",",
usecols=("A", "C", "E"), names=True)
assert_equal(test.dtype.names, ctrl_names)
#
test = np.genfromtxt(TextIO(data),
dtype=int, delimiter=",",
usecols=("A", "C", "E"), names=True)
assert_equal(test.dtype.names, ctrl_names) Example 12
def test_gft_using_filename(self):
# Test that we can load data from a filename as well as a file
# object
tgt = np.arange(6).reshape((2, 3))
if sys.version_info[0] >= 3:
# python 3k is known to fail for '\r'
linesep = ('\n', '\r\n')
else:
linesep = ('\n', '\r\n', '\r')
for sep in linesep:
data = '0 1 2' + sep + '3 4 5'
with temppath() as name:
with open(name, 'w') as f:
f.write(data)
res = np.genfromtxt(name)
assert_array_equal(res, tgt) Example 13
def get_image_data_and_labels(index_file, get_full_path=True, as_list=True):
if not os.path.exists(index_file):
print 'Error, no index file at path ', index_file
return [],[]
index_file_dir = os.path.dirname(index_file)
data = np.genfromtxt(index_file, dtype='str')
labels = data[:,1].astype(int)
if as_list:
im_data= list(data[:,0])
else:
im_data = data[:,0]
if get_full_path:
im_data_f = [join(index_file_dir,im) for im in im_data ]
if not as_list:
im_data_f = np.array(im_data_f)
else:
im_data_f = im_data
return im_data_f,labels Example 14
def main():
drawXtremIOCharts()
# data = np.genfromtxt('xtremPerfStats.csv', dtype=float, delimiter=',', names=True)
# print data.dtype.names
# iops = plot.figure()
# iopsInit = len(iops.axes)
# bw = plot.figure()
# bwInit = len(bw.axes)
# latency = plot.figure()
# latencyInit = len(latency.axes)
# xCpu = plot.figure()
# xCpuInit = len(xCpu.axes)
# for name in data.dtype.names:
# if re.search('iops', name):
# drawPlots(data,iops,name,"IOPs",iopsInit+1)
# if re.search('bandwidth', name):
# drawPlots(data,bw,name,"Bandwidth, MB/s", bwInit+1)
# if re.search('latency', name):
# drawPlots(data,latency,name,"Latency, MicroSec", latencyInit+1)
# if re.search('SC', name):
# drawPlots(data,xCpu,name,"% CPU Utilization", xCpuInit+1)
# plot.show() Example 15
def rave(dr=5, usecols=None):
"""
NAME:
rave
PURPOSE:
Load the RAVE data
INPUT:
dr= (5) data release
usecols= (sequence, optional) indices to read from RAVE data
OUTPUT:
data table
HISTORY:
2016-09-12 - Written - Bovy (UofT)
"""
filePath, ReadMePath= path.ravePath(dr=dr)
if not os.path.exists(filePath):
download.rave(dr=dr)
if dr == 4:
data= astropy.io.ascii.read(filePath,readme=ReadMePath)
elif dr == 5:
if usecols:
data= numpy.genfromtxt(filePath,delimiter=',', names=True, usecols=usecols)
else:
data= numpy.genfromtxt(filePath,delimiter=',', names=True)
return data Example 16
def __init__(self, **kwargs):
logging.info('Crossword __init__: Initializing crossword...')
logging.debug('kwargs:', kwargs)
# Reading kwargs
self.setup = kwargs
self.rows = int(kwargs.get('n', 5))
self.cols = int(kwargs.get('m', 5))
self.words_file = str(kwargs.get('word_file', 'lemma.num.txt'))
self.sort = bool(kwargs.get('sort', False))
self.maximize_len = bool(kwargs.get('maximize_len', False))
self.repeat_words = bool(kwargs.get('repeat_words', False))
logging.debug('Crossword __init__: n={}, m={}, fname={}'.format(self.rows, self.cols, self.words_file))
# Loading words
logging.debug('Crossword __init__: Started loading words from {}'.format(self.words_file))
arr = np.genfromtxt(self.words_file, dtype='str', delimiter=' ')
self.words = arr[np.in1d(arr[:, 3], ['v', 'n', 'adv', 'a'])][:, 2].tolist()
# Number of words loaded
logging.debug('Crossword __init__: Number of words loaded: {}'.format(len(self.words)))
self.words = list(set(x for x in self.words if len(x) <= self.rows and len(x) <= self.cols))
if self.sort:
self.words = sorted(self.words, key=len, reverse=self.maximize_len)
# After filter logging
logging.debug('Crossword __init__: Number of words after filter: {}, maxlen = {}'.format(len(self.words), len(
max(self.words, key=len)))) Example 17
def test_gd():
'''
A gradient descent and linear regression example to solve y = mx + b equation
using gradient descent, m is slope, b is y-intercept
by Matt Nedrich
Source: http://spin.atomicobject.com/2014/06/24/gradient-descent-linear-regression/
'''
# read data
points = genfromtxt("data/spring.csv", delimiter=",")
# initial y-intercept guess
b0 = 0
# initial slope guess
m0 = 0
# number of iterations to perform the GD
n_iter = 1000
for i in range(n_iter):
# perform GD iterations
b0, m0 = step_gradient(b0, m0, points, 0.0001)
print("GD\ti=%d\tb=%f\tm=%f\te=%f\t(y=%f*x+%f)" %
(n_iter, b0, m0, compute_error(b0, m0, points), m0, b0)) Example 18
def create_LOFAR_configuration(antfile: str, meta: dict = None) -> Configuration:
""" Define from the LOFAR configuration file
:param antfile:
:param meta:
:return: Configuration
"""
antxyz = numpy.genfromtxt(antfile, skip_header=2, usecols=[1, 2, 3], delimiter=",")
nants = antxyz.shape[0]
assert antxyz.shape[1] == 3, "Antenna array has wrong shape %s" % antxyz.shape
anames = numpy.genfromtxt(antfile, dtype='str', skip_header=2, usecols=[0], delimiter=",")
mounts = numpy.repeat('XY', nants)
location = EarthLocation(x=[3826923.9] * u.m, y=[460915.1] * u.m, z=[5064643.2] * u.m)
fc = Configuration(location=location, names=anames, mount=mounts, xyz=antxyz, frame='global',
diameter=35.0)
return fc Example 19
def getCytoRNADataFromCsv(dataPath, batchesPath, batch1, batch2, trainPct = 0.8):
data = genfromtxt(dataPath, delimiter=',', skip_header=0)
batches = genfromtxt(batchesPath, delimiter=',', skip_header=0)
source = data[batches == batch1]
target = data[batches == batch2]
n_source = source.shape[0]
p = np.random.permutation(n_source)
cutPt = int(n_source * trainPct)
source_train = source[p[:cutPt]]
source_test = source[p[cutPt:]]
n_target = target.shape[0]
p = np.random.permutation(n_target)
cutPt = int(n_target * trainPct)
target_train = target[p[:cutPt]]
target_test = target[p[cutPt:]]
return source_train, source_test, target_train, target_test Example 20
def getCytoRNADataFromCsv(dataPath, batchesPath, batch1, batch2, trainPct = 0.8):
data = genfromtxt(dataPath, delimiter=',', skip_header=0)
batches = genfromtxt(batchesPath, delimiter=',', skip_header=0)
source = data[batches == batch1]
target = data[batches == batch2]
n_source = source.shape[0]
p = np.random.permutation(n_source)
cutPt = int(n_source * trainPct)
source_train = source[p[:cutPt]]
source_test = source[p[cutPt:]]
n_target = target.shape[0]
p = np.random.permutation(n_target)
cutPt = int(n_target * trainPct)
target_train = target[p[:cutPt]]
target_test = target[p[cutPt:]]
return source_train, source_test, target_train, target_test Example 21
def test_l1l2path():
X_file = 'data_c/X_200_100.csv'
Y_file = 'data_c/Y_200_100.csv'
X = np.genfromtxt(X_file)
Y = np.genfromtxt(Y_file)
mu = 1e-3
tau_range = np.logspace(-2,0,3)
k_max = 10000
tolerance = 1e-4
pc = pplus.PPlusConnection(debug=False, workers_servers = ('127.0.0.1',))
pc.submit(l1l2path_job,
args=(X, Y, mu, tau_range, k_max, tolerance),
modules=('numpy as np', 'ctypes'))
result_keys = pc.collect()
print result_keys
print("Done") Example 22
def __init__(self,Hs,d,slope):
Hs = float(Hs)
d = float(d)
slope = float(slope)
battjes = genfromtxt("battjes.csv",delimiter=',') #import table with normalized wave heights from Battjes&Groenendijk 2000, Wave height distribution on shallow foreshores
if Hs/d >= 0.78:
self.Hs = 0.78*d
else:
self.Htr = (0.35+5.8*1/slope)*d
# Hrms equation .59 The Rock Manual (page 359)
self.Hrms = (0.6725 + 0.2025*(Hs/d))*Hs
# calculate the normalised Htr
HtrNorm = self.Htr / self.Hrms
#find nearest to self.Htr in column 1 of Battjes. Choose the value immediately next to it.
index = int(HtrNorm / 0.05) + 1
if index > 60:
index = 60
#extract the relevant wave heights from Battjes table.
self.Hs = battjes[index,3] * self.Hrms
self.H2Percent = battjes[index,5] * self.Hrms
self.H1Percent = battjes[index,6] * self.Hrms
self.Hmax = battjes[index,7] * self.Hrms Example 23
def getCalibMatrix(dataPath, frameNum):
# load calibration data
# P0, P1, P2, P3, Tr_velo_to_cam, Tr_imu_to_velo
pathCalib = 'calib/{:0>6}.txt'.format(frameNum)
P_left = np.genfromtxt(pathCalib,dtype=None,usecols=range(1,13),skip_header=2,skip_footer=4).reshape(3,4) # 4x4
rect_3x3 = np.genfromtxt(pathCalib,dtype=None,usecols=range(1,10),skip_header=4,skip_footer=2).reshape(3,3) # 3x3
velo2cam_3x4 = np.genfromtxt(pathCalib,dtype=None,usecols=range(1,13),skip_header=5,skip_footer=1).reshape(3,4) # 4x4
rect = np.eye(4)
velo2cam = np.eye(4)
rect[:3,:3] =rect_3x3
velo2cam[:3, :3] = velo2cam_3x4[:3,:3]
velo2cam[:3, 3] = velo2cam_3x4[:3, 3]
return {'P_left':P_left,'rect':rect,'velo2cam':velo2cam} Example 24
def getCalibMatrix(dataPath, frameNum):
# load calibration data
# P0, P1, P2, P3, Tr_velo_to_cam, Tr_imu_to_velo
pathCalib = dataPath+'calib/{:0>6}.txt'.format(frameNum)
P_left = np.genfromtxt(pathCalib,dtype=None,usecols=range(1,13),skip_header=2,skip_footer=4).reshape(3,4) # 4x4
rect_3x3 = np.genfromtxt(pathCalib,dtype=None,usecols=range(1,10),skip_header=4,skip_footer=2).reshape(3,3) # 3x3
velo2cam_3x4 = np.genfromtxt(pathCalib,dtype=None,usecols=range(1,13),skip_header=5,skip_footer=1).reshape(3,4) # 4x4
rect = np.eye(4)
velo2cam = np.eye(4)
rect[:3,:3] =rect_3x3
velo2cam[:3, :3] = velo2cam_3x4[:3,:3]
velo2cam[:3, 3] = velo2cam_3x4[:3, 3]
return {'P_left':P_left,'rect':rect,'velo2cam':velo2cam} Example 25
def run(self, args, extra_args):
if args.output_file is not None:
output_file = os.path.realpath(args.output_file)
else:
output_file = os.path.realpath(args.input_protocol)
additional_files = []
if args.additional_files:
for file in args.additional_files:
additional_files.append(np.genfromtxt(file))
protocol = mdt.load_protocol(os.path.realpath(args.input_protocol))
context_dict = {name: protocol.get_column(name) for name in protocol.column_names}
exec(args.expr, {'np': np, 'files': additional_files}, context_dict)
for key in context_dict:
if is_scalar(context_dict[key]):
context_dict[key] = np.ones(protocol.length) * context_dict[key]
protocol = Protocol(context_dict)
mdt.write_protocol(protocol, output_file) Example 26
def with_added_column_from_file(self, name, file_name, multiplication_factor=1):
"""Create a copy of this protocol with the given column (loaded from a file) added to this protocol.
The given file can either contain a single value or one value per protocol line.
Args:
name (str): The name of the column to add.
file_name (str): The file to get the column from.
multiplication_factor (double): we might need to scale the data by a constant. For example,
if the data in the file is in ms we might need to scale it to seconds by multiplying with 1e-3
Returns:
self: for chaining
"""
columns = copy.copy(self._columns)
if name == 'g':
columns.update(get_g_columns(file_name))
for column_name in ('gx', 'gy', 'gz'):
columns[column_name] *= multiplication_factor
return Protocol(columns)
else:
data = np.genfromtxt(file_name)
data *= multiplication_factor
return self.with_new_column(name, data) Example 27
def load_edges(fpath, delimiter=None, has_header=False):
"""Load edges in CSV format as numpy ndarray of strings.
Args:
fpath (str): edges file
delimiter (str): alternative argument name for sep (default=None)
has_header (bool): True if has header row
Returns:
np.ndarray: array of edges
"""
if PANDAS_INSTALLED:
header = 'infer' if has_header else None
df = pd.read_csv(fpath, delimiter=delimiter, header=header)
edges = df.values
else:
logger.warning("Pandas not installed. Using numpy to load csv, which "
"is slower.")
header = 1 if has_header else 0
edges = np.genfromtxt(fpath, delimiter=delimiter, skip_header=header,
dtype=object)
return edges.astype('str') Example 28
def read_xyt_frame( n=1 ):
''' Load the xyt txt files:
x,y is the detector (x,y) coordinates
t is the time-encoder (when hitting the detector at that (x,y))
DATA_DIR is the data filefold path
DataPref is the data prefix
n is file number
the data name will be like: DATA_DIR/DataPref_0001.txt
return the histogram of the hitting event
'''
import numpy as np
ni = '%04d'%n
fp = DATA_DIR + DataPref + '%s.txt'%ni
data = np.genfromtxt( fp, skiprows=0)[:,2] #take the time encoder
td = np.histogram( data, bins= np.arange(11810) )[0] #do histogram
return td Example 29
def read_xyt_frame( n=1 ):
''' Load the xyt txt files:
x,y is the detector (x,y) coordinates
t is the time-encoder (when hitting the detector at that (x,y))
DATA_DIR is the data filefold path
DataPref is the data prefix
n is file number
the data name will be like: DATA_DIR/DataPref_0001.txt
return the histogram of the hitting event
'''
import numpy as np
ni = '%04d'%n
fp = DATA_DIR + DataPref + '%s.txt'%ni
data = np.genfromtxt( fp, skiprows=0)[:,2] #take the time encoder
td = np.histogram( data, bins= np.arange(11810) )[0] #do histogram
return td Example 30
def read_xyt_frame( n=1 ):
''' Load the xyt txt files:
x,y is the detector (x,y) coordinates
t is the time-encoder (when hitting the detector at that (x,y))
DATA_DIR is the data filefold path
DataPref is the data prefix
n is file number
the data name will be like: DATA_DIR/DataPref_0001.txt
return the histogram of the hitting event
'''
import numpy as np
ni = '%04d'%n
fp = DATA_DIR + DataPref + '%s.txt'%ni
data = np.genfromtxt( fp, skiprows=0)[:,2] #take the time encoder
td = np.histogram( data, bins= np.arange(11810) )[0] #do histogram
return td Example 31
def check_subjects(subjects_info):
"Ensure subjects are provided and their data exist."
if isinstance(subjects_info, str):
if not pexists(subjects_info):
raise IOError('path to subject list does not exist: {}'.format(subjects_info))
subjects_list = np.genfromtxt(subjects_info, dtype=str)
elif isinstance(subjects_info, collections.Iterable):
if len(subjects_info) < 1:
raise ValueError('Empty subject list.')
subjects_list = subjects_info
else:
raise ValueError('Invalid value provided for subject list. \n '
'Must be a list of paths, or path to file containing list of paths, one for each subject.')
subject_id_list = np.atleast_1d(subjects_list)
num_subjects = subject_id_list.size
if num_subjects < 1:
raise ValueError('Input subject list is empty.')
num_digits_id_size = len(str(num_subjects))
max_id_width = max(map(len, subject_id_list))
return subject_id_list, num_subjects, max_id_width, num_digits_id_size Example 32
def write_preprocessed_data(output_directory, cell_IDs, cell_stages, data, markers):
processed_data_path = path.join(output_directory, 'processed_data.tsv')
with open(processed_data_path, 'w') as f:
f.write('\t'.join(cell_IDs))
f.write('\n')
f.write('\t'.join(cell_stages))
f.write('\n')
np.savetxt(f, data.T, fmt = '%.6f', delimiter = '\t')
dataset = np.genfromtxt(processed_data_path, delimiter = '\t', dtype = str)
dataset = np.insert(dataset, 0, np.append(['Cell ID', 'Stage'],
markers), axis = 1)
with open(processed_data_path, 'w') as f:
np.savetxt(f, dataset, fmt = '%s', delimiter = '\t') Example 33
def test_skip_footer_with_invalid(self):
with warnings.catch_warnings():
warnings.filterwarnings("ignore")
basestr = '1 1\n2 2\n3 3\n4 4\n5 \n6 \n7 \n'
# Footer too small to get rid of all invalid values
assert_raises(ValueError, np.genfromtxt,
TextIO(basestr), skip_footer=1)
# except ValueError:
# pass
a = np.genfromtxt(
TextIO(basestr), skip_footer=1, invalid_raise=False)
assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
#
a = np.genfromtxt(TextIO(basestr), skip_footer=3)
assert_equal(a, np.array([[1., 1.], [2., 2.], [3., 3.], [4., 4.]]))
#
basestr = '1 1\n2 \n3 3\n4 4\n5 \n6 6\n7 7\n'
a = np.genfromtxt(
TextIO(basestr), skip_footer=1, invalid_raise=False)
assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.], [6., 6.]]))
a = np.genfromtxt(
TextIO(basestr), skip_footer=3, invalid_raise=False)
assert_equal(a, np.array([[1., 1.], [3., 3.], [4., 4.]])) Example 34
def test_commented_header(self):
# Check that names can be retrieved even if the line is commented out.
data = TextIO("""
#gender age weight
M 21 72.100000
F 35 58.330000
M 33 21.99
""")
# The # is part of the first name and should be deleted automatically.
test = np.genfromtxt(data, names=True, dtype=None)
ctrl = np.array([('M', 21, 72.1), ('F', 35, 58.33), ('M', 33, 21.99)],
dtype=[('gender', '|S1'), ('age', int), ('weight', float)])
assert_equal(test, ctrl)
# Ditto, but we should get rid of the first element
data = TextIO(b"""
# gender age weight
M 21 72.100000
F 35 58.330000
M 33 21.99
""")
test = np.genfromtxt(data, names=True, dtype=None)
assert_equal(test, ctrl) Example 35
def test_dtype_with_object(self):
# Test using an explicit dtype with an object
data = """ 1; 2001-01-01
2; 2002-01-31 """
ndtype = [('idx', int), ('code', np.object)]
func = lambda s: strptime(s.strip(), "%Y-%m-%d")
converters = {1: func}
test = np.genfromtxt(TextIO(data), delimiter=";", dtype=ndtype,
converters=converters)
control = np.array(
[(1, datetime(2001, 1, 1)), (2, datetime(2002, 1, 31))],
dtype=ndtype)
assert_equal(test, control)
ndtype = [('nest', [('idx', int), ('code', np.object)])]
try:
test = np.genfromtxt(TextIO(data), delimiter=";",
dtype=ndtype, converters=converters)
except NotImplementedError:
pass
else:
errmsg = "Nested dtype involving objects should be supported."
raise AssertionError(errmsg) Example 36
def test_replace_space(self):
# Test the 'replace_space' option
txt = "A.A, B (B), C:C\n1, 2, 3.14"
# Test default: replace ' ' by '_' and delete non-alphanum chars
test = np.genfromtxt(TextIO(txt),
delimiter=",", names=True, dtype=None)
ctrl_dtype = [("AA", int), ("B_B", int), ("CC", float)]
ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
assert_equal(test, ctrl)
# Test: no replace, no delete
test = np.genfromtxt(TextIO(txt),
delimiter=",", names=True, dtype=None,
replace_space='', deletechars='')
ctrl_dtype = [("A.A", int), ("B (B)", int), ("C:C", float)]
ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
assert_equal(test, ctrl)
# Test: no delete (spaces are replaced by _)
test = np.genfromtxt(TextIO(txt),
delimiter=",", names=True, dtype=None,
deletechars='')
ctrl_dtype = [("A.A", int), ("B_(B)", int), ("C:C", float)]
ctrl = np.array((1, 2, 3.14), dtype=ctrl_dtype)
assert_equal(test, ctrl) Example 37
def test_names_with_usecols_bug1636(self):
# Make sure we pick up the right names w/ usecols
data = "A,B,C,D,E\n0,1,2,3,4\n0,1,2,3,4\n0,1,2,3,4"
ctrl_names = ("A", "C", "E")
test = np.genfromtxt(TextIO(data),
dtype=(int, int, int), delimiter=",",
usecols=(0, 2, 4), names=True)
assert_equal(test.dtype.names, ctrl_names)
#
test = np.genfromtxt(TextIO(data),
dtype=(int, int, int), delimiter=",",
usecols=("A", "C", "E"), names=True)
assert_equal(test.dtype.names, ctrl_names)
#
test = np.genfromtxt(TextIO(data),
dtype=int, delimiter=",",
usecols=("A", "C", "E"), names=True)
assert_equal(test.dtype.names, ctrl_names) Example 38
def test_gft_using_filename(self):
# Test that we can load data from a filename as well as a file
# object
tgt = np.arange(6).reshape((2, 3))
if sys.version_info[0] >= 3:
# python 3k is known to fail for '\r'
linesep = ('\n', '\r\n')
else:
linesep = ('\n', '\r\n', '\r')
for sep in linesep:
data = '0 1 2' + sep + '3 4 5'
with temppath() as name:
with open(name, 'w') as f:
f.write(data)
res = np.genfromtxt(name)
assert_array_equal(res, tgt) Example 39
def read_file_to_np(self, file_name):
datatype = [('time',np.float32), ('ax',np.int16), ('ay',np.int16), ('az',np.int16),
('gx',np.int16), ('gy',np.int16), ('gz',np.int16),
('mx',np.int16), ('my',np.int16), ('mz',np.int16),
('time_diff', np.float32)]
data = np.genfromtxt(file_name, dtype=datatype, delimiter="\t")
data['time'] = data['time']-data['time'][0]
a = np.diff(data['time'])
time_diff_array = np.insert(a, 0, 0)
data['time_diff'] = time_diff_array
# ?????
data['mx'] = data['mx'] * 1.18359375
data['my'] = data['my'] * 1.19140625
data['mz'] = data['mz'] * 1.14453125
return data Example 40
def get_overlapping_files(self, path, ra, dec, width):
"""
This function ...
:param path to the directory with the images
:param ra:
:param dec:
:param width:
:return:
"""
# Generate the meta and then overlap file
meta_path, overlap_path = self.generate_meta_and_overlap_file(path, ra, dec, width)
# Load the overlap table
overlap_files = np.genfromtxt(overlap_path, skip_header=3, usecols=[32], dtype="S500")
# Return the names of the overlapping images
return overlap_files
# ----------------------------------------------------------------- Example 41
def get_overlapping_files(self, path, ra, dec, width):
"""
This function ...
:param path to the directory with the images
:param ra:
:param dec:
:param width:
:return:
"""
# Generate the meta and then overlap file
meta_path, overlap_path = self.generate_meta_and_overlap_file(path, ra, dec, width)
# Load the overlap table
overlap_files = np.genfromtxt(overlap_path, skip_header=3, usecols=[32], dtype="S500")
# Return the names of the overlapping images
return overlap_files
# ----------------------------------------------------------------- Example 42
def read_gpl(self):
dtype = [('waves',float),]+[('spec%i'%(i+1),float) for i in range(len(self.age))]
self.sed = np.genfromtxt(self.workdir+self.csp_output+'.spec',dtype=dtype)
age3, Q = np.genfromtxt(self.workdir+self.csp_output+'.3color', usecols=(0,5), unpack=True)
age4, M = np.genfromtxt(self.workdir+self.csp_output+'.4color', usecols=(0,6), unpack=True)
for x,age in zip(self.sed.dtype.names[1:],self.age):
self.sed[x] = self.sed[x] * 3.839e33
self.sed[x][self.sed["waves"] < 912.] = self.sed[x][self.sed["waves"] < 912.] * self.lyc_esc
log_age = np.log10(age*1e9)
diff = abs(age3 - log_age)
self.Q[x] = Q[diff == min(diff)][0]
diff = abs(age4 - log_age)
self.M_unnorm[x] = M[diff == min(diff)][0] Example 43
def main(opts):
vertices = np.genfromtxt('points.dat', delimiter=' ', skip_header=1)
npoints, dim = vertices.shape
assert dim == 3
faces = np.genfromtxt('indices.dat', delimiter=' ') # Generated from alpha_shape
# Create the mesh
cube = mesh.Mesh(np.zeros(faces.shape[0], dtype=mesh.Mesh.dtype))
for i, f in enumerate(faces):
for j in range(3):
cube.vectors[i][j] = vertices[f[j],:]
# Write the mesh to file
cube.save(opts.new_file_name) Example 44
def read_array(filename):
''' Read array and convert to 2d np arrays '''
array = np.genfromtxt(filename, dtype=float)
if len(array.shape)==1:
array = array.reshape( -1, 1 )
return array Example 45
def file_to_array (filename, verbose=False):
''' Converts a file to a list of list of STRING
It differs from np.genfromtxt in that the number of columns doesn't need to be constant'''
data =[]
with open(filename, "r") as data_file:
if verbose: print ("Reading {}...".format(filename))
lines = data_file.readlines()
if verbose: print ("Converting {} to correct array...".format(filename))
data = [lines[i].strip().split() for i in range (len(lines))]
del lines #djajetic 11.11.2015 questionable
return data Example 46
def load_iris():
try:
# Load Iris dataset from the sklearn.datasets package
from sklearn import datasets
from sklearn import decomposition
# Load Dataset
iris = datasets.load_iris()
X = iris.data
y = iris.target
labels = iris.target_names
# Reduce components by Principal Component Analysis from sklearn
X = decomposition.PCA(n_components=3).fit_transform(X)
except ImportError:
# Load Iris dataset manually
path = os.path.join('data', 'iris', 'iris.data')
iris_data = np.genfromtxt(path, dtype='str', delimiter=',')
X = iris_data[:, :4].astype(dtype=float)
y = np.ndarray((X.shape[0],), dtype=int)
# Create target vector y and corresponding labels
labels, idx = [], 0
for i, label in enumerate(iris_data[:, 4]):
label = label.split('-')[1]
if label not in labels:
labels.append(label); idx += 1
y[i] = idx - 1
# Reduce components by implemented Principal Component Analysis
X = PCA(X, 3)[0]
return X, y, labels Example 47
def read_model_table(modelfile):
'''
This reads a downloaded TRILEGAL model file.
'''
infd = gzip.open(modelfile)
model = np.genfromtxt(infd,names=True)
infd.close()
return model Example 48
def test_stats2():
"""Test stats2 func from fluxpart.util"""
data = "7 8 4\n6 1 3\n10 6 6\n6 7 3\n8 2 4"
dtype = [('v0', int), ('v1', int), ('v2', int)]
arr = np.genfromtxt(io.BytesIO(data.encode()), dtype=dtype)
ans = stats2(arr)
npt.assert_allclose(ans.ave_v0, 37 / 5)
npt.assert_allclose(ans.ave_v1, 24 / 5)
npt.assert_allclose(ans.ave_v2, 4)
npt.assert_allclose(ans.var_v0, 14 / 5)
npt.assert_allclose(ans.var_v1, 97 / 10)
npt.assert_allclose(ans.var_v2, 3 / 2)
npt.assert_allclose(ans.cov_v0_v1, 3 / 5)
npt.assert_allclose(ans.cov_v0_v2, 2)
npt.assert_allclose(ans.cov_v1_v0, ans.cov_v0_v1)
npt.assert_allclose(ans.cov_v1_v2, 1)
npt.assert_allclose(ans.cov_v2_v0, ans.cov_v0_v2)
npt.assert_allclose(ans.cov_v2_v1, ans.cov_v1_v2)
data = "7 8 4\n6 1 3\n10 6 6\n6 7 3\n8 2 4"
dtype = [('v0', int), ('v1', int), ('v2', int)]
arr = np.genfromtxt(io.BytesIO(data.encode()), dtype=dtype)
ans = stats2(arr, names=('v0', 'v2'))
npt.assert_allclose(ans.ave_v0, 37 / 5)
npt.assert_allclose(ans.ave_v2, 4)
npt.assert_allclose(ans.var_v0, 14 / 5)
npt.assert_allclose(ans.var_v2, 3 / 2)
npt.assert_allclose(ans.cov_v0_v2, 2)
npt.assert_allclose(ans.cov_v2_v0, ans.cov_v0_v2)
assert not hasattr(ans, 'ave_v1')
assert not hasattr(ans, 'var_v1')
assert not hasattr(ans, 'cov_v0_v1')
assert not hasattr(ans, 'cov_v1_v0')
assert not hasattr(ans, 'cov_v1_v2')
assert not hasattr(ans, 'cov_v2_v1') Example 49
def merge_results(sol,files):
model = get_model_type(sol)
save_where = '/Batch results/'
working_path = getcwd().replace("\\", "/")+"/"
save_path = working_path+save_where
print("\nChecking for longest csv file")
lengths = []
for f in files:
to_merge_temp = working_path+"/Results/%s/INV_%s-%s_%s.csv" %(f,sol.model,model,f)
headers_temp = np.genfromtxt(to_merge_temp, delimiter=",", dtype=str, skip_footer=1)
lengths.append(len(headers_temp))
to_merge_max = working_path+"/Results/%s/INV_%s-%s_%s.csv" %(files[lengths.index(max(lengths))],sol.model,model,files[lengths.index(max(lengths))])
headers = np.genfromtxt(to_merge_max, delimiter=",", dtype=str, skip_footer=1)
print("\nMerging csv files")
if not path.exists(save_path):
makedirs(save_path)
# to_merge = working_path+"/Results/%s/INV_%s_%s.csv" %(files[0],model,files[0])
# headers = np.genfromtxt(to_merge, delimiter=",", dtype=str, skip_footer=1)
merged_inv_results = np.zeros((len(files), len(headers)))
merged_inv_results.fill(np.nan)
for i, f in enumerate(files):
to_add = np.loadtxt(working_path+"/Results/%s/INV_%s-%s_%s.csv" %(f,sol.model,model,f), delimiter=",", skiprows=1)
merged_inv_results[i][:to_add.shape[0]] = to_add
rows = np.array(files, dtype=str)[:, np.newaxis]
hd = ",".join(["ID"] + list(headers))
np.savetxt(save_path+"Merged_%s-%s_%s_TO_%s.csv" %(sol.model,model,files[0],files[-1]), np.hstack((rows, merged_inv_results)), delimiter=",", header=hd, fmt="%s")
print("Batch file successfully saved in:\n", save_path) Example 50
def merge_results(sol,files):
model = get_model_type(sol)
save_where = '/Batch results/'
working_path = getcwd().replace("\\", "/")+"/"
save_path = working_path+save_where
print("\nChecking for longest csv file")
lengths = []
for f in files:
to_merge_temp = working_path+"/Results/%s/INV_%s-%s_%s.csv" %(f,sol.model,model,f)
headers_temp = np.genfromtxt(to_merge_temp, delimiter=",", dtype=str, skip_footer=1)
lengths.append(len(headers_temp))
to_merge_max = working_path+"/Results/%s/INV_%s-%s_%s.csv" %(files[lengths.index(max(lengths))],sol.model,model,files[lengths.index(max(lengths))])
headers = np.genfromtxt(to_merge_max, delimiter=",", dtype=str, skip_footer=1)
print("\nMerging csv files")
if not path.exists(save_path):
makedirs(save_path)
# to_merge = working_path+"/Results/%s/INV_%s_%s.csv" %(files[0],model,files[0])
# headers = np.genfromtxt(to_merge, delimiter=",", dtype=str, skip_footer=1)
merged_inv_results = np.zeros((len(files), len(headers)))
merged_inv_results.fill(np.nan)
for i, f in enumerate(files):
to_add = np.loadtxt(working_path+"/Results/%s/INV_%s-%s_%s.csv" %(f,sol.model,model,f), delimiter=",", skiprows=1)
merged_inv_results[i][:to_add.shape[0]] = to_add
rows = np.array(files, dtype=str)[:, np.newaxis]
hd = ",".join(["ID"] + list(headers))
np.savetxt(save_path+"Merged_%s-%s_%s_TO_%s.csv" %(sol.model,model,files[0],files[-1]), np.hstack((rows, merged_inv_results)), delimiter=",", header=hd, fmt="%s")
print("Batch file successfully saved in:\n", save_path) 