The following are code examples for showing how to use . They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don’t like. You can also save this page to your account.
Example 1
def main():
parser = generate_parser()
args = parser.parse_args()
infile1 = h5py.File(args.input1, 'r')
infile2 = h5py.File(args.input2, 'r')
resolutions = numpy.intersect1d(infile1['resolutions'][...], infile2['resolutions'][...])
chroms = numpy.intersect1d(infile2['chromosomes'][...], infile2['chromosomes'][...])
results = {}
data1 = load_data(infile1, chroms, resolutions)
data2 = load_data(infile2, chroms, resolutions)
infile1.close()
infile2.close()
results = {}
results[(args.input1.split('/')[-1].strip('.quasar'), args.input2.split('/')[-1].strip('.quasar'))] = correlate_samples(data1, data2)
for resolution in data1.keys():
for chromo in chroms:
plt.scatter(data1[resolution][chromo][1].flatten(),data2[resolution][chromo][1].flatten(),alpha=0.1,color='red')
plt.show()
plt.savefig(args.output+'.res'+str(resolution)+'.chr'+chromo+'.pdf') Example 2
def clean_data(self, df, is_with_MICE=0):
df = df.copy()
if df.isnull().sum().sum() > 0:
if is_with_MICE:
# Imputation using MICE
numerical_features_names = self.extract_numerical_features(df)
df.loc[:, tuple(numerical_features_names)] = self.estimate_by_mice(df[numerical_features_names])
else:
if any(tuple(df.columns == 'y')):
df = df.dropna()
else:
df = df.dropna(1)
TwoSigmaFinModTools._feature_names_num = pd.Series(data=np.intersect1d(
TwoSigmaFinModTools._feature_names_num.values, df.columns), dtype=object)
TwoSigmaFinModTools._numerical_feature_names = TwoSigmaFinModTools.extract_numerical_features(df)
return df Example 3
def RecursionTree(self, node, x, x_rows, y):
#????
if len(x_rows) <= 0:
return
#????
if node.HasChildren() == False:
y[x_rows] = node.GetLabel()
# logger.debug('predict ????:%d,x_rows:%s', node.GetLabel(),x_rows)
return
feature = node.GetSplitAttr()
rest_x_row = np.array(x_rows, dtype = np.int)
for (value, child) in node.GetChildren():
new_x_row = np.intersect1d(x_rows, np.where(x[:,feature] == value)[0])
rest_x_row = arraysetops.setxor1d(rest_x_row, new_x_row, True)
self.RecursionTree(child, x, new_x_row, y)
#???????????????????????
y[rest_x_row] = self.Classify(y, x_rows) Example 4
def uintersect1d(arr1, arr2, assume_unique=False):
"""Find the sorted unique elements of the two input arrays.
A wrapper around numpy.intersect1d that preserves units. All input arrays
must have the same units. See the documentation of numpy.intersect1d for
full details.
Examples
--------
>>> A = yt.YTArray([1, 2, 3], 'cm')
>>> B = yt.YTArray([2, 3, 4], 'cm')
>>> uintersect1d(A, B)
YTArray([ 2., 3.]) cm
"""
v = np.intersect1d(arr1, arr2, assume_unique=assume_unique)
v = validate_numpy_wrapper_units(v, [arr1, arr2])
return v Example 5
def uunion1d(arr1, arr2):
"""Find the union of two arrays.
A wrapper around numpy.intersect1d that preserves units. All input arrays
must have the same units. See the documentation of numpy.intersect1d for
full details.
Examples
--------
>>> A = yt.YTArray([1, 2, 3], 'cm')
>>> B = yt.YTArray([2, 3, 4], 'cm')
>>> uunion1d(A, B)
YTArray([ 1., 2., 3., 4.]) cm
"""
v = np.union1d(arr1, arr2)
v = validate_numpy_wrapper_units(v, [arr1, arr2])
return v Example 6
def clean_data(self, df):
df = df.copy()
is_with_MICE = 1
if df.isnull().sum().sum() > 0:
if is_with_MICE:
# Imputation using MICE
numerical_features_names = self.extract_numerical_features(df)
df.loc[:, tuple(numerical_features_names)] = self.estimate_by_mice(df[numerical_features_names])
else:
if any(tuple(df.columns == 'SalePrice')):
df = df.dropna()
else:
df = df.dropna(1)
HousePrices._feature_names_num = pd.Series(data=np.intersect1d(HousePrices._feature_names_num
.values, df.columns), dtype=object)
return df Example 7
def getNui(self):
# Generates Users who are trusted by user u and have rated product i
sz = len(self.R_train_ui) + len(self.R_test_ui)
for u, i in self.R_train_ui:
# Users who have rated product i
rat_u = self.R_train[np.where(self.R_train[:, 1] == i), 0]
# Users trusted by u
trust_u = self.W[np.where(self.W[:, 0] == u),1]
self.V[u, i] = np.intersect1d(rat_u, trust_u)
print u,i,self.V[u, i]
for u, i in self.R_test_ui:
# Users who have rated product i
rat_u = self.R_train[np.where(self.R_train[:, 1] == i), 0]
# Users trusted by u
trust_u = self.W[np.where(self.W[:, 0] == u),1]
self.V[u, i] = np.intersect1d(rat_u, trust_u)
print u,i,self.V[u, i] Example 8
def transform(self, y):
"""Transform labels to normalized encoding.
Parameters
----------
y : array-like of shape [n_samples]
Target values.
Returns
-------
y : array-like of shape [n_samples]
"""
y = column_or_1d(y, warn=True)
classes = np.unique(y)
if len(np.intersect1d(classes, self.classes_)) < len(classes):
diff = np.setdiff1d(classes, self.classes_)
self.classes_ = np.hstack((self.classes_, diff))
return np.searchsorted(self.classes_, y)[0] Example 9
def test_intersection(self):
# intersect with Int64Index
other = Index(np.arange(1, 6))
result = self.index.intersection(other)
expected = np.sort(np.intersect1d(self.index.values, other.values))
self.assert_numpy_array_equal(result, expected)
result = other.intersection(self.index)
expected = np.sort(np.asarray(np.intersect1d(self.index.values,
other.values)))
self.assert_numpy_array_equal(result, expected)
# intersect with increasing RangeIndex
other = RangeIndex(1, 6)
result = self.index.intersection(other)
expected = np.sort(np.intersect1d(self.index.values, other.values))
self.assert_numpy_array_equal(result, expected)
# intersect with decreasing RangeIndex
other = RangeIndex(5, 0, -1)
result = self.index.intersection(other)
expected = np.sort(np.intersect1d(self.index.values, other.values))
self.assert_numpy_array_equal(result, expected) Example 10
def getAccRel(sim,dst):
# for how many is the most similar one also the one with the smallest dst
# relaxed formulation: of the most similar ones (if more than one is equally similar) one is among those with smallest dst (if more than one is equally far away)
maxSim = numpy.max(sim, axis=1)
minDst = numpy.min(dst, axis=1)
nSamp = sim.shape[0]
nCorrect = 0.
nCorrectClass = 0.
for i in xrange(nSamp):
maxSimIdx, = (sim[i,:] == maxSim[i]).nonzero()
minDstIdx, = (dst[i,:] == minDst[i]).nonzero()
if len(numpy.intersect1d(maxSimIdx,minDstIdx, assume_unique=True)) > 0:
nCorrect += 1.
if numpy.min(sim[i,minDstIdx]) > -2.:
nCorrectClass += 1.
acc = nCorrect / nSamp
# classification accuracy. for how many percent is the closest from the correct class
classAcc = nCorrectClass / nSamp
return (acc,classAcc) Example 11
def find_neighbors(self,_quadlist):
import numpy as np
neighbors = np.array([])
edges = [self.vertex_ids[[0,1]],
self.vertex_ids[[1,2]],
self.vertex_ids[[2,3]],
self.vertex_ids[[3,0]]]
for e in edges:
has_vertex1 = np.where(_quadlist == e[0])[0]
has_vertex2 = np.where(_quadlist == e[1])[0]
same_edge = np.intersect1d(has_vertex1, has_vertex2)
neighbor = same_edge[same_edge != self.quad_id]
neighbors = np.append(neighbors, neighbor)
return neighbors.astype(int) Example 12
def resolve_inside_vertex(self, _local_v_id, _dc_quads):
# all quads connected to the vertex will be removed anyhow
delete_quads_list = self.manifold_vertex_quad_ids[_local_v_id]
new_quads_list = []
# change all references to old manifold vertex to references to the right child vertex
for q_id in delete_quads_list:
quad = _dc_quads[q_id]
tmp = quad.index(self.v_idx[_local_v_id])
new_quad = list(quad)
for child_id in range(2):
if np.intersect1d(quad, self.v_children_connection_idx[_local_v_id][child_id]).__len__() != 0:
new_quad[tmp] = self.v_children_idx[_local_v_id][child_id]
break
new_quads_list.append(new_quad)
return new_quads_list, delete_quads_list Example 13
def __init__(self, obs, est, minval=None):
# Check input
assert len(obs) == len(est), \
"obs and est need to have the same length. " \
"len(obs)=%d, len(est)=%d" % (len(obs), len(est))
# only remember those entries which have both valid observations
# AND estimates
ix = np.intersect1d(util._idvalid(obs, minval=minval),
util._idvalid(est, minval=minval))
self.n = len(ix)
if self.n == 0:
print("WARNING: No valid pairs of observed and "
"estimated available for ErrorMetrics!")
self.obs = np.array([])
self.est = np.array([])
else:
self.obs = obs[ix]
self.est = est[ix]
self.resids = self.est - self.obs Example 14
def inFootprint(self, pixels, nside=None):
"""
Open each valid filename for the set of pixels and determine the set
of subpixels with valid data.
"""
if numpy.isscalar(pixels): pixels = numpy.array([pixels])
if nside is None: nside = self.nside_likelihood
inside = numpy.zeros( len(pixels), dtype='bool')
if not self.nside_catalog:
catalog_pix = [0]
else:
catalog_pix = superpixel(pixels,nside,self.nside_catalog)
catalog_pix = numpy.intersect1d(catalog_pix,self.catalog_pixels)
for filenames in self.filenames[catalog_pix]:
#logger.debug("Loading %s"%filenames['mask_1'])
subpix_1,val_1 = ugali.utils.skymap.readSparseHealpixMap(filenames['mask_1'],'MAGLIM',construct_map=False)
#logger.debug("Loading %s"%filenames['mask_2'])
subpix_2,val_2 = ugali.utils.skymap.readSparseHealpixMap(filenames['mask_2'],'MAGLIM',construct_map=False)
subpix = numpy.intersect1d(subpix_1,subpix_2)
superpix = numpy.unique(ugali.utils.skymap.superpixel(subpix,self.nside_pixel,nside))
inside |= numpy.in1d(pixels, superpix)
return inside Example 15
def getCatalogPixels(self):
"""
Return the catalog pixels spanned by this ROI.
"""
filenames = self.config.getFilenames()
nside_catalog = self.config.params['coords']['nside_catalog']
nside_pixel = self.config.params['coords']['nside_pixel']
# All possible catalog pixels spanned by the ROI
superpix = ugali.utils.skymap.superpixel(self.pixels,nside_pixel,nside_catalog)
superpix = numpy.unique(superpix)
# Only catalog pixels that exist in catalog files
pixels = numpy.intersect1d(superpix, filenames['pix'].compressed())
return pixels
############################################################ Example 16
def transform(self, y):
"""Transform labels to normalized encoding.
Parameters
----------
y : array-like of shape [n_samples]
Target values.
Returns
-------
y : array-like of shape [n_samples]
"""
check_is_fitted(self, 'classes_')
y = column_or_1d(y.ravel(), warn=True)
classes = np.unique(y)
if isinstance(classes[0], np.float64):
classes = classes[np.isfinite(classes)]
_check_numpy_unicode_bug(classes)
if len(np.intersect1d(classes, self.classes_)) < len(classes):
diff = np.setdiff1d(classes, self.classes_)
print(self.classes_)
raise ValueError("y contains new labels: %s" % str(diff))
return np.searchsorted(self.classes_, y).reshape(-1, 1) Example 17
def find_matching_indices(array, value_list):
assert isinstance(array, np.ndarray)
assert isinstance(value_list, np.ndarray)
# reimplemented in cython for speed # TODO !!! include in conda package
try:
from cython_tools import find_matching_indices_fast
return find_matching_indices_fast(array.astype('uint32'), value_list.astype('uint32'))
except ImportError:
print "WARNING: Could not find cython function, using slow numpy version"
indices = []
for i, row in enumerate(array):
if( np.intersect1d(row, value_list).size ):
indices.append(i)
return np.array(indices)
#
# Modified Adjacency
#
# TODO reactivate Example 18
def join(l1, l2): # join two sorted list # n1 = len(l1) # n2 = len(l2) # #l1.sort() # #l2.sort() # p1 = 0 # p2 = 0 # ret = [] # while p1 < n1 and p2 < n2: # if l1[p1] < l2[p2]: # p1 += 1 # elif l1[p1] > l2[p2]: # p2 += 1 # else: # ret.append(l1[p1]) # p1 += 1 # p2 += 1 return np.intersect1d(l1, l2)
Example 19
def add_observation(self, observation, indx=None):
"""
Parameters
----------
indx : ints
The indices of the healpixel map that have been observed by observation
"""
if observation['filter'][0] in self.filtername:
self.feature[indx] += 1
if self.mask_indx is not None:
overlap = np.intersect1d(indx, self.mask_indx)
if overlap.size > 0:
# interpolate over those pixels that are DD fields.
# XXX. Do I need to kdtree this? Maybe make a dict on init
# to lookup the N closest non-masked pixels, then do weighted average.
pass Example 20
def _relation_recall(gt_triplets, pred_triplets,
gt_boxes, pred_boxes, iou_thresh):
# compute the [email protected] metric for a set of predicted triplets
num_gt = gt_triplets.shape[0]
num_correct_pred_gt = 0
for gt, gt_box in zip(gt_triplets, gt_boxes):
keep = np.zeros(pred_triplets.shape[0]).astype(bool)
for i, pred in enumerate(pred_triplets):
if gt[0] == pred[0] and gt[1] == pred[1] and gt[2] == pred[2]:
keep[i] = True
if not np.any(keep):
continue
boxes = pred_boxes[keep,:]
sub_iou = iou(gt_box[:4], boxes[:,:4])
obj_iou = iou(gt_box[4:], boxes[:,4:])
inds = np.intersect1d(np.where(sub_iou >= iou_thresh)[0],
np.where(obj_iou >= iou_thresh)[0])
if inds.size > 0:
num_correct_pred_gt += 1
return float(num_correct_pred_gt) / float(num_gt) Example 21
def replaceCompWithExpansion(self, uid=0, xSS=None,
keysToSetNonExtraZero=['sumLogPiRemVec']):
''' Replace existing component with expanded set of statistics.
Post Condition
--------------
Values associated with uid are removed.
All entries of provided xSS are added last in index order.
'''
if not np.intersect1d(xSS.uids, self.uids).size == 0:
raise ValueError("Cannot expand with same uids.")
for key in self._Fields._FieldDims:
if key in keysToSetNonExtraZero:
arr = getattr(self._Fields, key)
arr.fill(0)
if hasattr(xSS, 'mUIDPairs'):
assert not self.hasMergeTerms()
self.setMergeUIDPairs(xSS.mUIDPairs)
self.insertComps(xSS)
self.removeComp(uid=uid) Example 22
def _stc_src_sel(src, stc):
""" Select the vertex indices of a source space using a source estimate
"""
if isinstance(stc, VolSourceEstimate):
vertices = [stc.vertices]
else:
vertices = stc.vertices
if not len(src) == len(vertices):
raise RuntimeError('Mismatch between number of source spaces (%s) and '
'STC vertices (%s)' % (len(src), len(vertices)))
src_sels = []
offset = 0
for s, v in zip(src, vertices):
src_sel = np.intersect1d(s['vertno'], v)
src_sel = np.searchsorted(s['vertno'], src_sel)
src_sels.append(src_sel + offset)
offset += len(s['vertno'])
src_sel = np.concatenate(src_sels)
return src_sel Example 23
def fix_predictions(self, X, predictions, bias):
idxs_users_missing, idxs_items_missing = self.indices_missing
# Set average when neither the user nor the item exist
g_avg = bias['globalAvg']
common_indices = np.intersect1d(idxs_users_missing, idxs_items_missing)
predictions[common_indices] = g_avg
# Only users exist (return average + {dUser})
if 'dUsers' in bias:
missing_users = np.setdiff1d(idxs_users_missing, common_indices)
if len(missing_users) > 0:
user_idxs = X[missing_users, self.order[0]]
predictions[missing_users] = g_avg + bias['dUsers'][user_idxs]
# Only items exist (return average + {dItem})
if 'dItems' in bias:
missing_items = np.setdiff1d(idxs_items_missing, common_indices)
if len(missing_items) > 0:
item_idxs = X[missing_items, self.order[1]]
predictions[missing_items] = g_avg + bias['dItems'][item_idxs]
return predictions Example 24
def test_shuffle_kfold():
# Check the indices are shuffled properly
kf = KFold(3)
kf2 = KFold(3, shuffle=True, random_state=0)
kf3 = KFold(3, shuffle=True, random_state=1)
X = np.ones(300)
all_folds = np.zeros(300)
for (tr1, te1), (tr2, te2), (tr3, te3) in zip(
kf.split(X), kf2.split(X), kf3.split(X)):
for tr_a, tr_b in combinations((tr1, tr2, tr3), 2):
# Assert that there is no complete overlap
assert_not_equal(len(np.intersect1d(tr_a, tr_b)), len(tr1))
# Set all test indices in successive iterations of kf2 to 1
all_folds[te2] = 1
# Check that all indices are returned in the different test folds
assert_equal(sum(all_folds), 300) Example 25
def test_stratified_shuffle_split_iter():
ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
np.array([-1] * 800 + [1] * 50)
]
for y in ys:
sss = StratifiedShuffleSplit(6, test_size=0.33,
random_state=0).split(np.ones(len(y)), y)
for train, test in sss:
assert_array_equal(np.unique(y[train]), np.unique(y[test]))
# Checks if folds keep classes proportions
p_train = (np.bincount(np.unique(y[train],
return_inverse=True)[1]) /
float(len(y[train])))
p_test = (np.bincount(np.unique(y[test],
return_inverse=True)[1]) /
float(len(y[test])))
assert_array_almost_equal(p_train, p_test, 1)
assert_equal(y[train].size + y[test].size, y.size)
assert_array_equal(np.lib.arraysetops.intersect1d(train, test), []) Example 26
def test_stratified_shuffle_split_iter():
ys = [np.array([1, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]),
np.array([0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3]),
np.array([0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2]),
np.array([1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4]),
np.array([-1] * 800 + [1] * 50)
]
for y in ys:
sss = cval.StratifiedShuffleSplit(y, 6, test_size=0.33,
random_state=0)
for train, test in sss:
assert_array_equal(np.unique(y[train]), np.unique(y[test]))
# Checks if folds keep classes proportions
p_train = (np.bincount(np.unique(y[train], return_inverse=True)[1])
/ float(len(y[train])))
p_test = (np.bincount(np.unique(y[test], return_inverse=True)[1])
/ float(len(y[test])))
assert_array_almost_equal(p_train, p_test, 1)
assert_equal(y[train].size + y[test].size, y.size)
assert_array_equal(np.intersect1d(train, test), []) Example 27
def transform(self, y):
"""Transform labels to normalized encoding.
Parameters
----------
y : array-like of shape [n_samples]
Target values.
Returns
-------
y : array-like of shape [n_samples]
"""
check_is_fitted(self, 'classes_')
y = column_or_1d(y, warn=True)
classes = np.unique(y)
_check_numpy_unicode_bug(classes)
if len(np.intersect1d(classes, self.classes_)) < len(classes):
diff = np.setdiff1d(classes, self.classes_)
raise ValueError("y contains new labels: %s" % str(diff))
return np.searchsorted(self.classes_, y) Example 28
def common_ids(descendent_ids, ancestor_ids, threshold=0.5):
r"""
Determine if at least a given fraction of ancestor's member particles
are in the descendent.
Parameters
----------
descendent_ids : list of ints
Member ids for first halo.
ancestor_ids : list of int
Member ids for second halo.
threshold : float, optional
Critical fraction of ancestor's particles ending up in the
descendent to be considered a true ancestor. Default: 0.5.
Returns
-------
True or False
"""
common = np.intersect1d(descendent_ids, ancestor_ids)
return common.size > threshold * ancestor_ids.size Example 29
def _nn_pose_fill(valid):
"""
Looks up closest True for each False and returns
indices for fill-in-lookup
In: [True, False, True, ... , False, True]
Out: [0, 0, 2, ..., 212, 212]
"""
valid_inds, = np.where(valid)
invalid_inds, = np.where(~valid)
all_inds = np.arange(len(valid))
all_inds[invalid_inds] = -1
for j in range(10):
fwd_inds = valid_inds + j
bwd_inds = valid_inds - j
# Forward fill
invalid_inds, = np.where(all_inds < 0)
fwd_fill_inds = np.intersect1d(fwd_inds, invalid_inds)
all_inds[fwd_fill_inds] = all_inds[fwd_fill_inds-j]
# Backward fill
invalid_inds, = np.where(all_inds < 0)
if not len(invalid_inds): break
bwd_fill_inds = np.intersect1d(bwd_inds, invalid_inds)
all_inds[bwd_fill_inds] = all_inds[bwd_fill_inds+j]
# Check if any missing
invalid_inds, = np.where(all_inds < 0)
if not len(invalid_inds): break
# np.set_printoptions(threshold=np.nan)
# print valid.astype(np.int)
# print np.array_str(all_inds)
# print np.where(all_inds < 0)
return all_inds Example 30
def __indexs_select_pk0(self,pk0_roi0_h0,pk0_roi0_h1,pk0_roi1_h0,pk0_roi1_h1):
# get indexs of selected waveforms in pk0
spk_in_line = np.apply_along_axis(self.__in_select_line,1,self.waveforms_pk0,pk0_roi0_h0,pk0_roi0_h1)
changed_index = np.where(spk_in_line==True)[0]
changed_index = np.array(changed_index,dtype=np.int32)
spk_in_line1 = np.apply_along_axis(self.__in_select_line,1,self.waveforms_pk0,pk0_roi1_h0,pk0_roi1_h1)
changed_index1 = np.where(spk_in_line1==True)[0]
changed_index1 = np.array(changed_index1,dtype=np.int32)
changed_index = np.intersect1d(changed_index, changed_index1)
return changed_index + self.indexs_pk0[0] Example 31
def __indexs_select_pk0(self,pk0_roi0_h0,pk0_roi0_h1,pk0_roi1_h0,pk0_roi1_h1):
# get indexs of selected waveforms in pk0
spk_in_line = np.apply_along_axis(self.__in_select_line,1,self.waveforms_pk0,pk0_roi0_h0,pk0_roi0_h1)
changed_index = np.where(spk_in_line==True)[0]
changed_index = np.array(changed_index,dtype=np.int32)
spk_in_line1 = np.apply_along_axis(self.__in_select_line,1,self.waveforms_pk0,pk0_roi1_h0,pk0_roi1_h1)
changed_index1 = np.where(spk_in_line1==True)[0]
changed_index1 = np.array(changed_index1,dtype=np.int32)
changed_index = np.intersect1d(changed_index, changed_index1)
return changed_index + self.indexs_pk0[0] Example 32
def intersect_and_sort_samples(sample_metadata, feature_table):
'''Return input tables retaining only shared samples, row order equivalent.
Parameters
----------
sample_metadata : pd.DataFrame
Contingency table with rows, columns = samples, metadata.
feature_table : pd.DataFrame
Contingency table with rows, columns = samples, features.
Returns
-------
sample_metadata, feature_table : pd.DataFrame, pd.DataFrame
Input tables with unshared samples removed and ordered equivalently.
Raises
------
ValueError
If no shared samples are found.
'''
shared_samples = np.intersect1d(sample_metadata.index, feature_table.index)
if shared_samples.size == 0:
raise ValueError('There are no shared samples between the feature '
'table and the sample metadata. Ensure that you have '
'passed the correct files.')
elif (shared_samples.size == sample_metadata.shape[0] ==
feature_table.shape[0]):
s_metadata = sample_metadata.copy()
s_features = feature_table.copy()
else:
s_metadata = sample_metadata.loc[np.in1d(sample_metadata.index,
shared_samples), :].copy()
s_features = feature_table.loc[np.in1d(feature_table.index,
shared_samples), :].copy()
return s_metadata, s_features.loc[s_metadata.index, :] Example 33
def intersect_sim(array_1, array_2):
"""Calculate the simiarity of two arrays
by using intersection / union
"""
sim = float(np.intersect1d(array_1, array_2).size) / \
float(np.union1d(array_1, array_2).size)
return sim Example 34
def split_data_in_epochs(self, data, epoch_length_sec, stride_sec):
"""
Split the signal in no-dropout-epochs of fixed length
"""
sig = np.array(data, dtype=np.float32) # [240000 x 16]
sig_epochs = []
samples_in_epoch = epoch_length_sec * SAMPLING_FREQUENCY
stride_shift = stride_sec * SAMPLING_FREQUENCY
# compute dropout indices (dropouts are at the same position across all channels)
drop_indices_c0 = np.where(sig[:,0]==0)[0]
drop_indices_c1 = np.where(sig[:,1]==0)[0]
drop_indices = np.intersect1d(drop_indices_c0, drop_indices_c1)
drop_indices = np.append(drop_indices, len(sig)) # add the index of the last element
window_start = 0
for window_end in drop_indices:
epoch_start = window_start
epoch_end = epoch_start + samples_in_epoch
while(epoch_end < window_end):
sig_epochs.append(sig[epoch_start:epoch_end, :])
epoch_start += stride_shift
epoch_end += stride_shift
window_start = window_end + 1
return(sig_epochs) Example 35
def Inversion(Qsca,Qabs,wavelength,diameter,nMin=1,nMax=3,kMin=0.001,kMax=1,scatteringPrecision=0.010,absorptionPrecision=0.010,spaceSize=120,interp=2):
error = lambda measured,calculated: np.abs((calculated-measured)/measured)
nRange = np.linspace(nMin,nMax,spaceSize)
kRange = np.logspace(np.log10(kMin),np.log10(kMax),spaceSize)
scaSpace = np.zeros((spaceSize,spaceSize))
absSpace = np.zeros((spaceSize,spaceSize))
for ni,n in enumerate(nRange):
for ki,k in enumerate(kRange):
_derp = fastMieQ(n+(1j*k),wavelength,diameter)
scaSpace[ni][ki] = _derp[0]
absSpace[ni][ki] = _derp[1]
if interp is not None:
nRange = zoom(nRange,interp)
kRange = zoom(kRange,interp)
scaSpace = zoom(scaSpace,interp)
absSpace = zoom(absSpace,interp)
scaSolutions = np.where(np.logical_and(Qsca*(1-scatteringPrecision)<scaSpace, scaSpace<Qsca*(1+scatteringPrecision)))
absSolutions = np.where(np.logical_and(Qabs*(1-absorptionPrecision)<absSpace, absSpace<Qabs*(1+absorptionPrecision)))
validScattering = nRange[scaSolutions[0]]+1j*kRange[scaSolutions[1]]
validAbsorption = nRange[absSolutions[0]]+1j*kRange[absSolutions[1]]
solution = np.intersect1d(validScattering,validAbsorption)
# errors = [error()]
return solution Example 36
def Inversion_SD(Bsca,Babs,wavelength,dp,ndp,nMin=1,nMax=3,kMin=0,kMax=1,scatteringPrecision=0.001,absorptionPrecision=0.001,spaceSize=40,interp=2):
dp = coerceDType(dp)
ndp = coerceDType(ndp)
nRange = np.linspace(nMin,nMax,spaceSize)
kRange = np.linspace(kMin,kMax,spaceSize)
scaSpace = np.zeros((spaceSize,spaceSize))
absSpace = np.zeros((spaceSize,spaceSize))
for ni,n in enumerate(nRange):
for ki,k in enumerate(kRange):
_derp = fastMie_SD(n+(1j*k),wavelength,dp,ndp)
scaSpace[ni][ki] = _derp[0]
absSpace[ni][ki] = _derp[1]
if interp is not None:
nRange = zoom(nRange,interp)
kRange = zoom(kRange,interp)
scaSpace = zoom(scaSpace,interp)
absSpace = zoom(absSpace,interp)
scaSolutions = np.where(np.logical_and(Bsca*(1-scatteringPrecision)<scaSpace, scaSpace<Bsca*(1+scatteringPrecision)))
absSolutions = np.where(np.logical_and(Babs*(1-absorptionPrecision)<absSpace, absSpace<Babs*(1+absorptionPrecision)))
validScattering = nRange[scaSolutions[0]]+1j*kRange[scaSolutions[1]]
validAbsorption = nRange[absSolutions[0]]+1j*kRange[absSolutions[1]]
return np.intersect1d(validScattering,validAbsorption) Example 37
def Inversion(Qsca,Qabs,wavelength,diameter,nMin=1,nMax=3,kMin=0.001,kMax=1,scatteringPrecision=0.010,absorptionPrecision=0.010,spaceSize=120,interp=2):
error = lambda measured,calculated: np.abs((calculated-measured)/measured)
nRange = np.linspace(nMin,nMax,spaceSize)
kRange = np.logspace(np.log10(kMin),np.log10(kMax),spaceSize)
scaSpace = np.zeros((spaceSize,spaceSize))
absSpace = np.zeros((spaceSize,spaceSize))
for ni,n in enumerate(nRange):
for ki,k in enumerate(kRange):
_derp = fastMieQ(n+(1j*k),wavelength,diameter)
scaSpace[ni][ki] = _derp[0]
absSpace[ni][ki] = _derp[1]
if interp is not None:
nRange = zoom(nRange,interp)
kRange = zoom(kRange,interp)
scaSpace = zoom(scaSpace,interp)
absSpace = zoom(absSpace,interp)
scaSolutions = np.where(np.logical_and(Qsca*(1-scatteringPrecision)<scaSpace, scaSpace<Qsca*(1+scatteringPrecision)))
absSolutions = np.where(np.logical_and(Qabs*(1-absorptionPrecision)<absSpace, absSpace<Qabs*(1+absorptionPrecision)))
validScattering = nRange[scaSolutions[0]]+1j*kRange[scaSolutions[1]]
validAbsorption = nRange[absSolutions[0]]+1j*kRange[absSolutions[1]]
solution = np.intersect1d(validScattering,validAbsorption)
# errors = [error()]
return solution Example 38
def Inversion_SD(Bsca,Babs,wavelength,dp,ndp,nMin=1,nMax=3,kMin=0,kMax=1,scatteringPrecision=0.001,absorptionPrecision=0.001,spaceSize=40,interp=2):
dp = coerceDType(dp)
ndp = coerceDType(ndp)
nRange = np.linspace(nMin,nMax,spaceSize)
kRange = np.linspace(kMin,kMax,spaceSize)
scaSpace = np.zeros((spaceSize,spaceSize))
absSpace = np.zeros((spaceSize,spaceSize))
for ni,n in enumerate(nRange):
for ki,k in enumerate(kRange):
_derp = fastMie_SD(n+(1j*k),wavelength,dp,ndp)
scaSpace[ni][ki] = _derp[0]
absSpace[ni][ki] = _derp[1]
if interp is not None:
nRange = zoom(nRange,interp)
kRange = zoom(kRange,interp)
scaSpace = zoom(scaSpace,interp)
absSpace = zoom(absSpace,interp)
scaSolutions = np.where(np.logical_and(Bsca*(1-scatteringPrecision)<scaSpace, scaSpace<Bsca*(1+scatteringPrecision)))
absSolutions = np.where(np.logical_and(Babs*(1-absorptionPrecision)<absSpace, absSpace<Babs*(1+absorptionPrecision)))
validScattering = nRange[scaSolutions[0]]+1j*kRange[scaSolutions[1]]
validAbsorption = nRange[absSolutions[0]]+1j*kRange[absSolutions[1]]
return np.intersect1d(validScattering,validAbsorption) Example 39
def _get_cluster_indices(self):
self.clusters = _np.intersect1d(
_np.where(self.density > self.min_density)[0],
_np.where(self.delta > self.min_delta)[0], assume_unique=True).astype(_np.intc)
self.nclusters = self.clusters.shape[0] Example 40
def _get_cluster_indices(self):
self.clusters = _np.intersect1d(
_np.where(self.density > self.min_density)[0],
_np.where(self.delta > self.min_delta)[0], assume_unique=True)
self.ncl = self.clusters.shape[0] Example 41
def select_regoin(img, vert, keep_shape=True, qmask=None, ):
'''Get a pixellist by a rectangular region
defined by
verts e.g. xs,xe,ys,ye = vert #x_start, x_end, y_start,y_end
(dimy, dimx,) = img.shape
Giving cut postion, start, end, width '''
import numpy as np
xs,xe,ys,ye = vert
if keep_shape:
img_= np.zeros_like( img )
#img_= np.zeros( [dimy,dimx])
try:
img_[ys:ye, xs:xe] = True
except:
img_[ys:ye, xs:xe,:] = True
pixellist_ = np.where( img_.ravel() )[0]
#pixellist_ = img_.ravel()
if qmask is not None:
b=np.where( qmask.flatten()==False )[0]
pixellist_ = np.intersect1d(pixellist_,b)
#imgx = img[pixellist_]
#imgx = imgx.reshape( xe-xs, ye-ys)
imgx = img_.ravel()
imgx[pixellist_] = img.ravel()[pixellist_]
imgx = imgx.reshape( img.shape )
else:
try:
imgx =img[ys:ye, xs:xe]
except:
imgx =img[ys:ye, xs:xe,:]
return imgx Example 42
def read_crop(self,varname,kt):
z2d = zeros((len(self.yidx),len(self.xidx)))
time = self.nc[0].variables['t'][kt]
#print('nt=%i / kt=%i'%(self.nt,kt))
if (kt>=self.nt):
print('kt is out of range max(kt)=%i'%(self.nt-1))
else:
for proc in range(self.nbproc):
varloc = self.nc[proc].variables[varname]
print('\r %12s - kt =%i - proc = %i'%(varname,kt,proc),end='')
ip,jp = proc%self.npx,proc//self.npx
iiglo = arange( ip*self.nxproc,(ip+1)*self.nxproc )
jjglo = arange( jp*self.nyproc,(jp+1)*self.nyproc )
ii = intersect1d( self.xidx, iiglo)
jj = intersect1d( self.yidx, jjglo)
if( (len(ii)>0) & (len(jj)>0)):
i0,i1=ii[0]-ip*self.nxproc,ii[-1]-ip*self.nxproc+1
j0,j1=jj[0]-jp*self.nyproc,jj[-1]-jp*self.nyproc+1
zz = varloc[kt,j0:j1,i0:i1]
i0,i1=ii[0]-self.xidx[0], ii[-1]+1-self.xidx[0]
j0,j1=jj[0]-self.yidx[0], jj[-1]+1-self.yidx[0]
z2d[j0:j1,i0:i1] = zz
return time,z2d Example 43
def get_data_stats(datasets):
data_stats_cols = ['all', 'non-fraud', 'fraud']
data_stats = pd.DataFrame(columns=data_stats_cols)
data_stats.loc['transactions'] = [d.shape[0] for d in datasets]
data_stats.loc['transactions/hour'] = [round(d['Local_Date'].apply(lambda x: x.hour).value_counts().sum()/24/366, 2) for d in datasets]
data_stats.loc['transactions/day'] = [round(d['Local_Date'].apply(lambda x: x.day).value_counts().sum() / 366, 2) for d in datasets]
data_stats.loc['transactions/week'] = [round(d['Local_Date'].apply(lambda x: x.week).value_counts().sum() / 52, 2) for d in datasets]
data_stats.loc['transactions/month'] = [round(d['Local_Date'].apply(lambda x: x.month).value_counts().sum() / 12, 2) for d in datasets]
data_stats.loc['cards'] = [len(d["CardID"].unique()) for d in datasets]
data_stats.loc['cards, single use'] = [sum(d["CardID"].value_counts() == 1) for d in datasets]
data_stats.loc['cards, multi use'] = [sum(d["CardID"].value_counts() > 1) for d in datasets]
cards_genuine = datasets[1]['CardID'].unique()
cards_fraud = datasets[2]['CardID'].unique()
data_stats.loc['fraud cards in genuine'] = ['-', '-', len(np.intersect1d(cards_genuine, cards_fraud)) / len(cards_fraud)]
data_stats.loc['first transaction'] = [min(d["Global_Date"]).date() for d in datasets]
data_stats.loc['last transaction'] = [max(d["Global_Date"]).date() for d in datasets]
data_stats.loc['min amount'] = [min(d["Amount"]) for d in datasets]
data_stats.loc['max amount'] = [max(d["Amount"]) for d in datasets]
data_stats.loc['avg amount'] = [np.average(d["Amount"]) for d in datasets]
data_stats.loc['num merchants'] = [len(d["MerchantID"].unique()) for d in datasets]
data_stats.loc['countries'] = [len(d["Country"].unique()) for d in datasets]
data_stats.loc['currencies'] = [len(d["Currency"].unique()) for d in datasets]
data_stats.loc['min trans/card'] = [min(d["CardID"].value_counts()) for d in datasets]
data_stats.loc['max trans/card'] = [max(d["CardID"].value_counts()) for d in datasets]
data_stats.loc['avg trans/card'] = [np.average(d["CardID"].value_counts()) for d in datasets]
return data_stats Example 44
def test_numpy_wrappers():
a1 = YTArray([1, 2, 3], 'cm')
a2 = YTArray([2, 3, 4, 5, 6], 'cm')
catenate_answer = [1, 2, 3, 2, 3, 4, 5, 6]
intersect_answer = [2, 3]
union_answer = [1, 2, 3, 4, 5, 6]
assert_array_equal(YTArray(catenate_answer, 'cm'), uconcatenate((a1, a2)))
assert_array_equal(catenate_answer, np.concatenate((a1, a2)))
assert_array_equal(YTArray(intersect_answer, 'cm'), uintersect1d(a1, a2))
assert_array_equal(intersect_answer, np.intersect1d(a1, a2))
assert_array_equal(YTArray(union_answer, 'cm'), uunion1d(a1, a2))
assert_array_equal(union_answer, np.union1d(a1, a2)) Example 45
def test_subhalos():
ds = data_dir_load(g298)
total_sub = 0
total_int = 0
for hid in range(0, ds.index.particle_count["Group"]):
my_h = ds.halo("Group", hid)
h_ids = my_h["ID"]
for sid in range(int(my_h["subhalo_number"][0])):
my_s = ds.halo("Subhalo", (my_h.particle_identifier, sid))
total_sub += my_s["ID"].size
total_int += np.intersect1d(h_ids, my_s["ID"]).size
# Test that all subhalo particles are contained within
# their parent group.
assert_equal(total_sub, total_int) Example 46
def test_boolean_spheres_overlap():
r"""Test to make sure that boolean objects (spheres, overlap)
behave the way we expect.
Test overlapping spheres.
"""
ds = fake_amr_ds()
sp1 = ds.sphere([0.45, 0.45, 0.45], 0.15)
sp2 = ds.sphere([0.55, 0.55, 0.55], 0.15)
# Get indices of both.
i1 = sp1["index","morton_index"]
i2 = sp2["index","morton_index"]
# Make some booleans
bo1 = sp1 & sp2
bo2 = sp1 - sp2
bo3 = sp1 | sp2
bo4 = ds.union([sp1, sp2])
bo5 = ds.intersection([sp1, sp2])
# Now make sure the indices also behave as we expect.
lens = np.intersect1d(i1, i2)
apple = np.setdiff1d(i1, i2)
both = np.union1d(i1, i2)
b1 = bo1["index","morton_index"]
b1.sort()
b2 = bo2["index","morton_index"]
b2.sort()
b3 = bo3["index","morton_index"]
b3.sort()
assert_array_equal(b1, lens)
assert_array_equal(b2, apple)
assert_array_equal(b3, both)
b4 = bo4["index","morton_index"]
b4.sort()
b5 = bo5["index","morton_index"]
b5.sort()
assert_array_equal(b3, b4)
assert_array_equal(b1, b5)
bo6 = sp1 ^ sp2
b6 = bo6["index", "morton_index"]
b6.sort()
assert_array_equal(b6, np.setxor1d(i1, i2)) Example 47
def test_boolean_ellipsoids_overlap():
r"""Test to make sure that boolean objects (ellipsoids, overlap)
behave the way we expect.
Test overlapping ellipsoids.
"""
ds = fake_amr_ds()
ell1 = ds.ellipsoid([0.45]*3, 0.05, 0.05, 0.05, np.array([0.1]*3), 0.1)
ell2 = ds.ellipsoid([0.55]*3, 0.05, 0.05, 0.05, np.array([0.1]*3), 0.1)
# Get indices of both.
i1 = ell1["index","morton_index"]
i2 = ell2["index","morton_index"]
# Make some booleans
bo1 = ell1 & ell2
bo2 = ell1 - ell2
bo3 = ell1 | ell2
bo4 = ds.union([ell1, ell2])
bo5 = ds.intersection([ell1, ell2])
# Now make sure the indices also behave as we expect.
overlap = np.intersect1d(i1, i2)
diff = np.setdiff1d(i1, i2)
both = np.union1d(i1, i2)
b1 = bo1["index","morton_index"]
b1.sort()
b2 = bo2["index","morton_index"]
b2.sort()
b3 = bo3["index","morton_index"]
b3.sort()
assert_array_equal(b1, overlap)
assert_array_equal(b2, diff)
assert_array_equal(b3, both)
b4 = bo4["index","morton_index"]
b4.sort()
b5 = bo5["index","morton_index"]
b5.sort()
assert_array_equal(b3, b4)
assert_array_equal(b1, b5)
bo6 = ell1 ^ ell2
b6 = bo6["index", "morton_index"]
b6.sort()
assert_array_equal(b6, np.setxor1d(i1, i2)) Example 48
def find_relative_parentage(self, child):
# Return two values: percent this halo gave to the other, and percent
# of the other that comes from this halo
overlap = np.intersect1d(self.particle_ids, child.particle_ids).size
of_child_from_me = float(overlap)/child.particle_ids.size
of_mine_from_me = float(overlap)/self.particle_ids.size
return of_child_from_me, of_mine_from_me Example 49
def _nonzero_intersection(m, m_hat):
'''Count the number of nonzeros in and between m and m_hat.
Returns
----------
m_nnz : number of nonzeros in m (w/o diagonal)
m_hat_nnz : number of nonzeros in m_hat (w/o diagonal)
intersection_nnz : number of nonzeros in intersection of m/m_hat
(w/o diagonal)
'''
n_features, _ = m.shape
m_no_diag = m.copy()
m_no_diag[np.diag_indices(n_features)] = 0
m_hat_no_diag = m_hat.copy()
m_hat_no_diag[np.diag_indices(n_features)] = 0
m_hat_nnz = len(np.nonzero(m_hat_no_diag.flat)[0])
m_nnz = len(np.nonzero(m_no_diag.flat)[0])
intersection_nnz = len(np.intersect1d(
np.nonzero(m_no_diag.flat)[0],
np.nonzero(m_hat_no_diag.flat)[0]
))
return m_nnz, m_hat_nnz, intersection_nnz Example 50
def _count_support_diff(m, m_hat):
n_features, _ = m.shape
m_no_diag = m.copy()
m_no_diag[np.diag_indices(n_features)] = 0
m_hat_no_diag = m_hat.copy()
m_hat_no_diag[np.diag_indices(n_features)] = 0
m_nnz = len(np.nonzero(m_no_diag.flat)[0])
m_hat_nnz = len(np.nonzero(m_hat_no_diag.flat)[0])
nnz_intersect = len(np.intersect1d(np.nonzero(m_no_diag.flat)[0],
np.nonzero(m_hat_no_diag.flat)[0]))
return m_nnz + m_hat_nnz - (2 * nnz_intersect) 