The following are code examples for showing how to use . They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don’t like. You can also save this page to your account.
Example 1
def save(self, filename):
"""Saves the collection to a file.
Parameters
----------
filename : :obj:`str`
The file to save the collection to.
Raises
------
ValueError
If the file extension is not .npy or .npz.
"""
file_root, file_ext = os.path.splitext(filename)
if file_ext == '.npy':
np.save(filename, self._data)
elif file_ext == '.npz':
np.savez_compressed(filename, self._data)
else:
raise ValueError('Extension %s not supported for point saves.' %(file_ext)) Example 2
def save_weights(fname, params, metadata=None):
""" assumes all params have unique names.
"""
# Includes batchnorm params now
names = [par.name for par in params]
if len(names) != len(set(names)):
raise ValueError('need unique param names')
param_dict = { param.name : param.get_value(borrow=False)
for param in params }
if metadata is not None:
param_dict['metadata'] = pickle.dumps(metadata)
logging.info('saving {} parameters to {}'.format(len(params), fname))
# try to avoid half-written files
fname = Path(fname)
if fname.exists():
tmp_fname = Path(fname.stripext() + '.tmp.npz') # TODO yes, this is a hack
np.savez_compressed(str(tmp_fname), **param_dict)
tmp_fname.rename(fname)
else:
np.savez_compressed(str(fname), **param_dict) Example 3
def save(self, NumpyFile):
# open file
nfile = open(NumpyFile, "wb")
# save internals
numpy.savez_compressed(nfile,
_ImgNum = numpy.array([self._ImgNum]),
_MeasNum = numpy.array([self._MeasNum]),
_KeySizeBits = numpy.array([self._KeySizeBits]),
_KeySizeBytes = numpy.array([self._KeySizeBytes]),
_KeyData = self._KeyData,
_ImageInfo = self._ImageInfo,
*[self._ImageData[idx] for idx in range(0, self._ImgNum)])
# close file
nfile.close() Example 4
def process_training_data(num_clips):
"""
Processes random training clips from the full training data. Saves to TRAIN_DIR_CLIPS by
default.
@param num_clips: The number of clips to process. Default = 5000000 (set in __main__).
@warning: This can take a couple of hours to complete with large numbers of clips.
"""
num_prev_clips = len(glob(c.TRAIN_DIR_CLIPS + '*'))
for clip_num in xrange(num_prev_clips, num_clips + num_prev_clips):
clip = process_clip()
np.savez_compressed(c.TRAIN_DIR_CLIPS + str(clip_num), clip)
if (clip_num + 1) % 100 == 0: print 'Processed %d clips' % (clip_num + 1) Example 5
def main():
"""
Commandline interface to extract parameters.
"""
log_sockeye_version(logger)
params = argparse.ArgumentParser(description="Extract specific parameters.")
arguments.add_extract_args(params)
args = params.parse_args()
if os.path.isdir(args.input):
param_path = os.path.join(args.input, C.PARAMS_BEST_NAME)
else:
param_path = args.input
ext_params = extract(param_path, args.names, args.list_all)
if len(ext_params) > 0:
utils.check_condition(args.output != None, "An output filename must be specified. (Use --output)")
logger.info("Writting extracted parameters to '%s'", args.output)
np.savez_compressed(args.output, **ext_params) Example 6
def save_vocab(self, path=None):
""" Saves the vocabulary into a file.
# Arguments:
path: Where the vocabulary should be saved. If not specified, a
randomly generated filename is used instead.
"""
dtype = ([('word', '|S{}'.format(self.word_length_limit)), ('count', 'int')])
np_dict = np.array(self.word_counts.items(), dtype=dtype)
# sort from highest to lowest frequency
np_dict[::-1].sort(order='count')
data = np_dict
if path is None:
path = str(uuid.uuid4())
np.savez_compressed(path, data=data)
print("Saved dict to {}".format(path)) Example 7
def savetofile(self, outfile):
"""Save model parameters to file."""
# Pickle non-matrix params into bytestring, then convert to numpy byte array
pklbytes = pickle.dumps({'hyper': self.hyper, 'epoch': self.epoch, 'pos': self.pos},
protocol=pickle.HIGHEST_PROTOCOL)
p = np.fromstring(pklbytes, dtype=np.uint8)
# Gather parameter matrices and names
pvalues = { n:m.get_value() for n, m in self.params.items() }
# Now save params and matrices to file
try:
np.savez_compressed(outfile, p=p, **pvalues)
except OSError as e:
raise e
else:
if isinstance(outfile, str):
stdout.write("Saved model parameters to {0}\n".format(outfile)) Example 8
def save_npz(filename, obj, compression=True):
"""Saves an object to the file in NPZ format.
This is a short-cut function to save only one object into an NPZ file.
Args:
filename (str): Target file name.
obj: Object to be serialized. It must support serialization protocol.
compression (bool): If ``True``, compression in the resulting zip file
is enabled.
"""
s = DictionarySerializer()
s.save(obj)
with open(filename, 'wb') as f:
if compression:
numpy.savez_compressed(f, **s.target)
else:
numpy.savez(f, **s.target) Example 9
def export_trimmed_glove_vectors(vocab, glove_filename, trimmed_filename, dim):
"""Saves glove vectors in numpy array
Args:
vocab: dictionary vocab[word] = index
glove_filename: a path to a glove file
trimmed_filename: a path where to store a matrix in npy
dim: (int) dimension of embeddings
"""
embeddings = np.zeros([len(vocab), dim])
with open(glove_filename) as f:
for line in f:
line = line.strip().split(' ')
word = line[0]
embedding = [float(x) for x in line[1:]]
if word in vocab:
word_idx = vocab[word]
embeddings[word_idx] = np.asarray(embedding)
np.savez_compressed(trimmed_filename, embeddings=embeddings) Example 10
def export_trimmed_glove_vectors(vocab, glove_filename, trimmed_filename, dim):
"""
Saves glove vectors in numpy array
Args:
vocab: dictionary vocab[word] = index
glove_filename: a path to a glove file
trimmed_filename: a path where to store a matrix in npy
dim: (int) dimension of embeddings
"""
embeddings = np.zeros([len(vocab), dim])
with open(glove_filename,encoding="utf-8") as f:
for line in f:
line = line.strip().split()
word = line[0]
embedding = map(float, line[1:])
if word in vocab:
word_idx = vocab[word]
embeddings[word_idx] = np.asarray(list(embedding))
np.savez_compressed(trimmed_filename, embeddings=embeddings) Example 11
def storeData(df, fileLoc='./tmp/', cv=0.30, rs=21):
"""
# Store the train and CV data in the tmp location for the classifiers.
# Input: df: Transformed DataFrame of the Adult Dataset.
# fileLoc: location of tmp where the binary data will be stored.
# cv: ratio of the cross_validation set in train-cv split
# rs: random_state used to the split.
# returns: None
# Note: data can be accessed using:
# Ex: data = np.load('./tmp/testTrainData.npz')
# and access the train/test using split using dictionary formatting.
# Ex: data['XTrain']
"""
if not os.path.exists('tmp'):
os.makedirs('tmp')
filename = fileLoc+'testTrainData'
XTrain, XTest, yTrain, yTest = trainCvSplit(df, cv, rs)
kwargs = {'XTrain': XTrain,
'XTest': XTest,
'yTrain': yTrain,
'yTest': yTest
}
np.savez_compressed(filename, **kwargs)
return None Example 12
def _make_npz(path, urls):
x_url, y_url = urls
x_path = download.cached_download(x_url)
y_path = download.cached_download(y_url)
with gzip.open(x_path, 'rb') as fx, gzip.open(y_path, 'rb') as fy:
fx.read(4)
fy.read(4)
N, = struct.unpack('>i', fx.read(4))
if N != struct.unpack('>i', fy.read(4))[0]:
raise RuntimeError('wrong pair of MNIST images and labels')
fx.read(8)
x = numpy.empty((N, 784), dtype=numpy.uint8)
y = numpy.empty(N, dtype=numpy.uint8)
for i in six.moves.range(N):
y[i] = ord(fy.read(1))
for j in six.moves.range(784):
x[i, j] = ord(fx.read(1))
numpy.savez_compressed(path, x=x, y=y)
return {'x': x, 'y': y} Example 13
def load(filename, n_episodes_model=1):
""" Load model (T,R) from <filename>_model.npz. Update t, r, s0
if no model is available, generate and save from SASR_step file """
global t, r, s0
file_model = filename + ".npz"
if os.path.isfile(file_model):
print("Model file found")
with np.load(file_model) as fm:
t = fm['T']
r = fm['R']
s0 = fm['s0']
else:
print("Model file not found")
generate_t_and_r(filename, n_episodes_model) # create t, r, s0
""" Save model (T,R) to <filename>_model.npz """
np.savez_compressed(file_model, T=t, R=r, s0=s0)
return Example 14
def save_mean_representations(model, model_filename, X, labels, pred_file):
n_items, dv = X.shape
n_classes = model.n_classes
n_topics = model.d_t
# try normalizing input vectors
test_X = normalize(np.array(X, dtype='float32'), axis=1)
model.load_params(model_filename)
# evaluate bound on test set
item_mus = []
for item in range(n_items):
y = labels[item]
# save the mean document representation
r_mu = model.get_mean_doc_rep(test_X[item, :], y)
item_mus.append(np.array(r_mu))
# write all the test doc representations to file
if pred_file is not None and n_topics > 1:
np.savez_compressed(pred_file, X=np.array(item_mus), y=labels) Example 15
def save(self, filename):
"""Writes the image to a file.
Parameters
----------
filename : :obj:`str`
The file to save the image to. Must be one of .png, .jpg,
.npy, or .npz.
Raises
------
ValueError
If an unsupported file type is specified.
"""
file_root, file_ext = os.path.splitext(filename)
if file_ext in COLOR_IMAGE_EXTS:
im_data = self._image_data()
pil_image = PImage.fromarray(im_data.squeeze())
pil_image.save(filename)
elif file_ext == '.npy':
np.save(filename, self._data)
elif file_ext == '.npz':
np.savez_compressed(filename, self._data)
else:
raise ValueError('Extension %s not supported' % (file_ext)) Example 16
def write_npz(windows, proximity_matrix, output_file):
"""Write a proximity matrix to an npz file.
npz files are a compressed numpy-specific format, meaning
they take up less disk space, but cannot be easily opened
by other programming languages (e.g. R). For more information
see :func:`numpy.savez_compressed`.
:param tuple windows: (list of x-axis windows, list of y-axis windows)
:param proximity_matrix: Input proximity matrix.
:type proximity_matrix: :class:`numpy array <numpy.ndarray>`
:param str filepath: Path to save matrix file.
"""
window_dict = {
'windows_{}'.format(i): win for i,
win in enumerate(windows)}
np.savez_compressed(output_file, scores=proximity_matrix, **window_dict) Example 17
def SaveGeometryMatrix(self,filename='geo_matrix'):
if ((self.LMatrixColumns is not None) &
(self.LMatrixRows is not None) &
(self.LMatrixValues is not None)):
np.savez_compressed(filename, \
columns = self.LMatrixColumns,\
rows = self.LMatrixRows,\
values = self.LMatrixValues,\
shape = self.LMatrixShape, \
grid_rmin = self.Rmin,\
grid_rmax = self.Rmax,\
grid_nr = self.nR,\
grid_zmin = self.Zmin,\
grid_zmax = self.Zmax,\
grid_nz = self.nZ,\
gridtype = 'RectangularGeometryMatrix') Example 18
def save_weights(self,filename): # save both weights and variables
with open(filename,'wb') as f:
# extract all weights in one go:
w = self.get_value_of(self.get_weights()+self.traverse('variables'))
print(len(w),'weights (and variables) obtained.')
# create an array object and put all the arrays into it.
# otherwise np.asanyarray() within np.savez_compressed()
# might make stupid mistakes
arrobj = np.empty([len(w)],dtype='object') # array object
for i in range(len(w)):
arrobj[i] = w[i]
np.savez_compressed(f,w=arrobj)
print('successfully saved to',filename)
return True Example 19
def run(self):
while True:
name, data = self.queue.get()
if name is None:
break
if data.shape[2] == 1 or data.shape[2] == 3:
name += '.png'
cv2.imwrite(os.path.join(self.path + name), data)
#imgOut = cv2.resize(imgOut, dsize=(img.shape[1],img.shape[0]))
#original[:,:,0] = np.repeat(np.mean(original, axis=2, keepdims=True), 3, axis=2)
#original[:,:,0] *= 1-imgOut* 1.3
#original[:,:,1] *= 1-imgOut* 1.3
#original[:,:,2] *= imgOut* 1.3
#cv2.imshow('OUT2', original /255)
#cv2.waitKey(1)
#cv2.imwrite('%s-shown.png' % fileName, original)
else:
name += '.npz'
np.savez_compressed(os.path.join(self.path + name), data=data) Example 20
def install(
self, local_dst_dir_=None, local_src_dir_=None, clean_install_=False):
'''
Install the dataset into directly usable format,
requires downloading for public dataset.
Args:
local_dst_dir_: string or None
where to install the dataset, None -> "%(default_dir)s"
local_src_dir_: string or None
where to find the raw downloaded files, None -> "%(default_dir)s"
'''
local_dst_dir = self.DEFAULT_DIR if local_dst_dir_ is None else Path(local_dst_dir_)
local_src_dir = self.DEFAULT_DIR if local_src_dir_ is None else Path(local_src_dir_)
local_dst_dir.mkdir(parents=True, exist_ok=True)
assert local_src_dir.exists()
images = np.empty((60000,3,32,32), dtype=np.uint8)
labels = np.empty((60000,), dtype=np.uint8)
tarfile_name = str(local_src_dir / 'cifar-10-python.tar.gz')
with tarfile.open(tarfile_name, 'r:gz') as tf:
for i in range(5):
with tf.extractfile('cifar-10-batches-py/data_batch_%d'%(i+1)) as f:
data_di = pickle.load(f, encoding='bytes')
images[(10000*i):(10000*(i+1))] = data_di[b'data'].reshape((10000,3,32,32))
labels[(10000*i):(10000*(i+1))] = np.asarray(data_di[b'labels'], dtype=np.uint8)
with tf.extractfile('cifar-10-batches-py/test_batch') as f:
data_di = pickle.load(f, encoding='bytes')
images[50000:60000] = data_di[b'data'].reshape((10000,3,32,32))
labels[50000:60000] = data_di[b'labels']
np.savez_compressed(str(local_dst_dir / 'cifar10.npz'), images=images, labels=labels)
if clean_install_:
os.remove(tarfile_name) Example 21
def write_sar_log(sars: List, logdir: str, episode_reward: int, suffix: str=''):
"""Write state-action-rewards to a log file."""
np.savez_compressed(os.path.join(logdir,
'%s_%s%s' % (str(time.time())[-5:], episode_reward, suffix)), np.vstack(sars)) Example 22
def install(
self, local_dst_dir_=None, local_src_dir_=None, clean_install_=False):
'''
Install the dataset into directly usable format,
requires downloading for public dataset.
Args:
local_dst_dir_: string or None
where to install the dataset, None -> "%(default_dir)s"
local_src_dir_: string or None
where to find the raw downloaded files, None -> "%(default_dir)s"
'''
local_dst_dir = self.DEFAULT_DIR if local_dst_dir_ is None else Path(local_dst_dir_)
local_src_dir = self.DEFAULT_DIR if local_src_dir_ is None else Path(local_src_dir_)
local_dst_dir.mkdir(parents=True, exist_ok=True)
assert local_src_dir.exists()
images = np.empty((60000,3,32,32), dtype=np.uint8)
labels = np.empty((60000,), dtype=np.uint8)
tarfile_name = str(local_src_dir / 'cifar-10-python.tar.gz')
with tarfile.open(tarfile_name, 'r:gz') as tf:
for i in range(5):
with tf.extractfile('cifar-10-batches-py/data_batch_%d'%(i+1)) as f:
data_di = pickle.load(f, encoding='bytes')
images[(10000*i):(10000*(i+1))] = data_di[b'data'].reshape((10000,3,32,32))
labels[(10000*i):(10000*(i+1))] = np.asarray(data_di[b'labels'], dtype=np.uint8)
with tf.extractfile('cifar-10-batches-py/test_batch') as f:
data_di = pickle.load(f, encoding='bytes')
images[50000:60000] = data_di[b'data'].reshape((10000,3,32,32))
labels[50000:60000] = data_di[b'labels']
np.savez_compressed(str(local_dst_dir / 'cifar10.npz'), images=images, labels=labels)
if clean_install_:
os.remove(tarfile_name) Example 23
def train_glove(infile, inputSize=20000, batchSize=100, dimensionSize=100, maxEpochs=1000, outfile='result', x_max=100, alpha=0.75): options = locals().copy() print 'initializing parameters' params = init_params(options) tparams = init_tparams(params) print 'loading data' I, J, Weight = load_data(infile) n_batches = int(np.ceil(float(I.get_value(borrow=True).shape[0]) / float(batchSize))) print 'building models' weightVector, iVector, jVector, cost = build_model(tparams, options) grads = T.grad(cost, wrt=tparams.values()) f_grad_shared, f_update = adadelta(tparams, grads, weightVector, iVector, jVector, cost) logFile = outfile + '.log' print 'training start' for epoch in xrange(maxEpochs): costVector = [] iteration = 0 for batchIndex in random.sample(range(n_batches), n_batches): cost = f_grad_shared(Weight.get_value(borrow=True, return_internal_type=True)[batchIndex*batchSize:(batchIndex+1)*batchSize], I.get_value(borrow=True, return_internal_type=True)[batchIndex*batchSize: (batchIndex+1)*batchSize], J.get_value(borrow=True, return_internal_type=True)[batchIndex*batchSize: (batchIndex+1)*batchSize]) f_update() costVector.append(cost) if (iteration % 1000 == 0): buf = 'epoch:%d, iteration:%d/%d, cost:%f' % (epoch, iteration, n_batches, cost) print buf print2file(buf, logFile) iteration += 1 trainCost = np.mean(costVector) buf = 'epoch:%d, cost:%f' % (epoch, trainCost) print buf print2file(buf, logFile) tempParams = unzip(tparams) np.savez_compressed(outfile + '.' + str(epoch), **tempParams)
Example 24
def save_matrix(f, m):
np.savez_compressed(f, data=m.data, indices=m.indices, indptr=m.indptr, shape=m.shape) Example 25
def save_pkl_files(dsm_prefix, dsm, save_in_one_file=False):
"""
Save the space to separate pkl files.
:param dsm_prefix:
:param dsm:
"""
# Save in a single file (for small spaces)
if save_in_one_file:
io_utils.save(dsm, dsm_prefix + '.pkl')
# Save in multiple files: npz for the matrix and pkl for the other data members of Space
else:
mat = coo_matrix(dsm.cooccurrence_matrix.get_mat())
np.savez_compressed(dsm_prefix + 'cooc.npz', data=mat.data, row=mat.row, col=mat.col, shape=mat.shape)
with open(dsm_prefix + '_row2id.pkl', 'wb') as f_out:
pickle.dump(dsm._row2id, f_out, 2)
with open(dsm_prefix + '_id2row.pkl', 'wb') as f_out:
pickle.dump(dsm._id2row, f_out, 2)
with open(dsm_prefix + '_column2id.pkl', 'wb') as f_out:
pickle.dump(dsm._column2id, f_out, 2)
with open(dsm_prefix + '_id2column.pkl', 'wb') as f_out:
pickle.dump(dsm._id2column, f_out, 2) Example 26
def _close(self):
# Write everything
np.savez_compressed(self.request.get_file(), *self._images) Example 27
def test_compressed_roundtrip():
arr = np.random.rand(200, 200)
npz_file = os.path.join(tempdir, 'compressed.npz')
np.savez_compressed(npz_file, arr=arr)
arr1 = np.load(npz_file)['arr']
assert_array_equal(arr, arr1) Example 28
def savez_compressed(file, *args, **kwds):
"""Saves one or more arrays into a file in compressed ``.npz`` format.
It is equivalent to :func:`cupy.savez` function except the output file is
compressed.
.. seealso::
:func:`cupy.savez` for more detail,
:func:`numpy.savez_compressed`
"""
args = map(cupy.asnumpy, args)
for key in kwds:
kwds[key] = cupy.asnumpy(kwds[key])
numpy.savez_compressed(file, *args, **kwds) Example 29
def process_glove(args, vocab_list, save_path, size=4e5, random_init=True):
"""
:param vocab_list: [vocab]
:return:
"""
if not gfile.Exists(save_path + ".npz"):
glove_path = os.path.join(args.glove_dir, "glove.6B.{}d.txt".format(args.glove_dim))
if random_init:
glove = np.random.randn(len(vocab_list), args.glove_dim)
else:
glove = np.zeros((len(vocab_list), args.glove_dim))
found = 0
with open(glove_path, 'r') as fh:
for line in tqdm(fh, total=size):
array = line.lstrip().rstrip().split(" ")
word = array[0]
vector = list(map(float, array[1:]))
if word in vocab_list:
idx = vocab_list.index(word)
glove[idx, :] = vector
found += 1
if word.capitalize() in vocab_list:
idx = vocab_list.index(word.capitalize())
glove[idx, :] = vector
found += 1
if word.upper() in vocab_list:
idx = vocab_list.index(word.upper())
glove[idx, :] = vector
found += 1
print("{}/{} of word vocab have corresponding vectors in {}".format(found, len(vocab_list), glove_path))
np.savez_compressed(save_path, glove=glove)
print("saved trimmed glove matrix at: {}".format(save_path)) Example 30
def write_values(self, tensors, compress=False):
"""
write dictionary of numpy.ndarray's with Op name as key to file
Arguments:
tensors (dict): A dictionary of numpy.ndarray's with Op name as key
compress: specify whether to compress tensors
"""
if compress:
np.savez_compressed(self.name, **tensors)
else:
np.savez(self.name, **tensors) Example 31
def process_word2vec(word2vec_dir, vocab, save_path, random_init=True):
# read pre-trained word embedddings from the binary file
print('Loading google word2vec...')
word2vec_path = word2vec_dir + '/GoogleNews-vectors-negative300.bin.gz'
word_vectors = KeyedVectors.load_word2vec_format(word2vec_path, binary=True)
print('Word2vec loaded!')
if random_init:
word2vec = np.random.uniform(-0.25, 0.25, (len(vocab), 300))
else:
word2vec = np.zeros((len(vocab), 300))
found = 0
for idx, token in enumerate(vocab):
try:
vec = word_vectors[token]
except:
pass
else:
word2vec[idx, :] = vec
found += 1
del word_vectors
print("{}/{} of word vocab have corresponding vectors in {}".format(found, len(vocab), word2vec_path))
np.savez_compressed(save_path, word2vec=word2vec)
print("saved trimmed word2vec matrix at: {}".format(save_path))
# construct embedding vectors according to the GloVe word vectors and vocabulary Example 32
def process_glove(glove_dir, glove_dim, vocab_dir, save_path, random_init=True):
"""
:param vocab_list: [vocab]
:return:
"""
save_path = save_path + '.{}'.format(glove_dim)
if not os.path.isfile(save_path + ".npz"):
# read vocabulary
with open(vocab_dir + '/vocabulary.pickle', 'rb') as f:
vocab_map = cPickle.load(f)
f.close()
vocab_list = list(zip(*vocab_map)[0])
glove_path = os.path.join(glove_dir, "glove.6B.{}d.txt".format(glove_dim))
if random_init:
glove = np.random.uniform(-0.25, 0.25, (len(vocab_list), glove_dim))
else:
glove = np.zeros((len(vocab_list), glove_dim))
found = 0
with open(glove_path, 'r') as fh:
for line in fh.readlines():
array = line.lstrip().rstrip().split(" ")
word = array[0]
vector = list(map(float, array[1:]))
if word in vocab_list:
idx = vocab_list.index(word)
glove[idx, :] = vector
found += 1
if word.capitalize() in vocab_list:
idx = vocab_list.index(word.capitalize())
glove[idx, :] = vector
found += 1
if word.upper() in vocab_list:
idx = vocab_list.index(word.upper())
glove[idx, :] = vector
found += 1
print("{}/{} of word vocab have corresponding vectors in {}".format(found, len(vocab_list), glove_path))
np.savez_compressed(save_path, glove=glove)
print("saved trimmed glove matrix at: {}".format(save_path)) Example 33
def save_estimates(self, fname='', notes='', force=False):
"""
Saves the JIVE estimates
U, D, V, full, rank for block secific joint/individual spaces
U, D, V, rank for common joint space
some metadata (when saved, some nots)
Parameters
----------
fname: name of the file
notes: any notes you want to include
force: whether or note to overwrite a file with the same name
"""
if os.path.exists(fname) and (not force):
raise ValueError('%s already exists' % fname)
kwargs = {}
svd_dat = ['scores', 'sing_vals', 'loadings', 'rank']
kwargs['K'] = self.K
block_estimates = self.get_block_specific_estimates()
for k in range(self.K):
for mode in ['joint', 'individual']:
for dat in svd_dat + ['full']:
label = '%d_%s_%s' % (k, mode, dat)
kwargs[label] = block_estimates[k][mode][dat]
common_joint = self.get_common_joint_space_estimate()
for dat in svd_dat:
kwargs['common_%s' % dat] = common_joint[dat]
current_time = time.strftime("%m/%d/%Y %H:%M:%S")
kwargs['metadata'] = [current_time, notes]
np.savez_compressed(fname, **kwargs) Example 34
def save_init_svd(self, fname='', notes='', force=False):
"""
Saves the initial SVD so it can be loaded later without recomputing
Parameters
----------
fname: name of the file
notes: any notes you want to include
force: whether or note to overwrite a file with the same name
"""
if not hasattr(self.blocks[0], 'scores'):
raise ValueError('initial svd has not yet been computed')
if os.path.exists(fname) and (not force):
raise ValueError('%s already exists' % fname)
kwargs = {}
svd_dat = ['scores', 'sing_vals', 'loadings', 'rank']
kwargs['K'] = self.K
for k in range(self.K):
kwargs['%d_scores' % k] = self.blocks[k].scores
kwargs['%d_sv' % k] = self.blocks[k].sv
kwargs['%d_loadings' % k ] = self.blocks[k].loadings
kwargs['%d_init_svd_rank' % k] = self.blocks[k].init_svd_rank
np.savez_compressed(fname, **kwargs) Example 35
def save(self, out_file):
"""
Save the current memory into a file in Numpy format
:param out_file: File storage path
:return:
"""
np.savez_compressed(out_file, states=self._states, actions=self._actions,
rewards=self._rewards, terminals=self._terminals) Example 36
def _save_np_compressed_data(file_name, *args):
mkdirs_if_not_exist(dirname(file_name))
np.savez_compressed(file_name, *args) Example 37
def save(self, out_file):
"""
Save the current memory into a file in Numpy format
:param out_file: File storage path
:return:
"""
np.savez_compressed(out_file, states=self._states, actions=self._actions,
rewards=self._rewards, terminals=self._terminals) Example 38
def save_frame_data(archive, path, videos, object_point_set, verbose=True):
if verbose:
print("Saving corners to {0:s}".format(path))
for video in videos:
archive[IMAGE_POINTS + str(video.name)] = video.image_points
archive[FRAME_NUMBERS + str(video.name)] = list(video.usable_frames.keys())
if len(video.poses) > 0:
archive[POSES + str(video.name)] = np.array([pose.T for pose in video.poses])
archive[OBJECT_POINT_SET] = object_point_set
np.savez_compressed(path, **archive) Example 39
def save_calibration_intervals(archive, path, videos, verbose=True):
if verbose:
print("Saving calibration intervals to {0:s}".format(path))
ranges = []
for video in videos:
if video.calibration_interval is None:
raise ValueError("Expecting all cameras to have valid calibration frame ranges. Got: None")
ranges.append(video.calibration_interval)
ranges = np.array(ranges)
archive[CALIBRATION_INTERVALS] = ranges
np.savez_compressed(path, **archive) Example 40
def save_model(self):
logging.info("Saving model")
save_filename = os.path.join(self.model_folder,'{}_epoch{}.npz'.format(self.model_name, self.epoch))
np.savez_compressed(save_filename, *lasagne.layers.get_all_param_values(self.network)) Example 41
def _preprocess(self, input_file, tensor_file):
if input_file.endswith(".bz2"): file_reference = BZ2File(input_file, "r")
elif input_file.endswith(".txt"): file_reference = io.open(input_file, "r")
raw_data = file_reference.read()
file_reference.close()
data = raw_data.encode(encoding=self.encoding)
# Convert the entirety of the data file from characters to indices via the vocab dictionary.
# How? map(function, iterable) returns a list of the output of the function
# executed on each member of the iterable. E.g.:
# [14, 2, 9, 2, 0, 6, 7, 0, ...]
# np.array converts the list into a numpy array.
self.tensor = np.array(list(map(self.vocab.get, data)))
# Compress and save the numpy tensor array to data.npz.
np.savez_compressed(tensor_file, tensor_data=self.tensor) Example 42
def save_vocab(self, path_count, path_vocab, word_limit=100000):
""" Saves the master vocabulary into a file.
"""
# reserve space for 10 special tokens
words = OrderedDict()
for token in SPECIAL_TOKENS:
# store -1 instead of np.inf, which can overflow
words[token] = -1
# sort words by frequency
desc_order = OrderedDict(sorted(self.master_vocab.items(),
key=lambda kv: kv[1], reverse=True))
words.update(desc_order)
# use encoding of up to 30 characters (no token conversions)
# use float to store large numbers (we don't care about precision loss)
np_vocab = np.array(words.items(),
dtype=([('word', '|S30'), ('count', 'float')]))
# output count for debugging
counts = np_vocab[:word_limit]
np.savez_compressed(path_count, counts=counts)
# output the index of each word for easy lookup
final_words = OrderedDict()
for i, w in enumerate(words.keys()[:word_limit]):
final_words.update({w: i})
with open(path_vocab, 'w') as f:
f.write(json.dumps(final_words, indent=4, separators=(',', ': '))) Example 43
def test_compressed_roundtrip():
arr = np.random.rand(200, 200)
npz_file = os.path.join(tempdir, 'compressed.npz')
np.savez_compressed(npz_file, arr=arr)
arr1 = np.load(npz_file)['arr']
assert_array_equal(arr, arr1) Example 44
def save_weights(self, weightspath=None):
weightspath = super(LasagneNetwork, self)._weightspath(weightspath)
weights = {name: p.get_value() for name, p in
LasagneNetwork._get_named_params(self.out_layer)}
np.savez_compressed(weightspath, **weights) Example 45
def _preprocess(self, input_file, tensor_file):
if input_file.endswith(".bz2"): file_reference = BZ2File(input_file, "r")
elif input_file.endswith(".txt"): file_reference = io.open(input_file, "r")
raw_data = file_reference.read()
file_reference.close()
data = raw_data.encode(encoding=self.encoding)
# Convert the entirety of the data file from characters to indices via the vocab dictionary.
# How? map(function, iterable) returns a list of the output of the function
# executed on each member of the iterable. E.g.:
# [14, 2, 9, 2, 0, 6, 7, 0, ...]
# np.array converts the list into a numpy array.
self.tensor = np.array(map(self.vocab.get, data))
# Compress and save the numpy tensor array to data.npz.
np.savez_compressed(tensor_file, tensor_data=self.tensor) Example 46
def main(em_file, em_result):
'''
embedding ->numpy
'''
em = word2vec.load(em_file)
vec = (em.vectors)
word2id = em.vocab_hash
# d = dict(vector = vec, word2id = word2id)
# t.save(d,em_result)
np.savez_compressed(em_result,vector=vec,word2id=word2id) Example 47
def savelogs(self, ts=None, saveres=True, filename=None):
# FIXME: consider HDF5
if ts == None:
ts = time.strftime("%Y%m%d-%H%M%S")
# np.save("%s/log-x-%s" % (self.cfgprefix, ts), self.iosm.x_)
# np.save("%s/log-x_raw-%s" % (self.cfgprefix, ts), self.iosm.x_raw_)
# np.save("%s/log-z-%s" % (self.cfgprefix, ts), self.iosm.z_)
# np.save("%s/log-zn-%s" % (self.cfgprefix, ts), self.iosm.zn_)
# np.save("%s/log-zn_lp-%s" % (self.cfgprefix, ts), self.iosm.zn_lp_)
# np.save("%s/log-r-%s" % (self.cfgprefix, ts), self.iosm.r_)
# np.save("%s/log-w-%s" % (self.cfgprefix, ts), self.iosm.w_)
# network data, pickling reservoir, input weights, output weights
# self.res.save("%s/log-%s-res-%s.bin" % (self.cfgprefix, self.cfgprefix, ts))
if filename == None:
logfile = "%s/log-learner-%s" % (self.cfgprefix, ts)
else:
logfile = filename
if saveres:
np.savez_compressed(logfile, x = self.iosm.x_,
x_raw = self.iosm.x_raw_, z = self.iosm.z_, zn = self.iosm.zn_,
zn_lp = self.iosm.zn_lp_, r = self.iosm.r_, w = self.iosm.w_, e = self.iosm.e_,
t = self.iosm.t_, mse = self.iosm.mse_)
else:
np.savez_compressed(logfile, x = self.iosm.x_,
x_raw = self.iosm.x_raw_, z = self.iosm.z_, zn = self.iosm.zn_,
zn_lp = self.iosm.zn_lp_, w = self.iosm.w_, e = self.iosm.e_,
t = self.iosm.t_,
mse = self.iosm.mse_)
print "logs saved to %s" % logfile
return logfile Example 48
def save_matrix(self, path):
with open(path, 'w') as f:
np.savez_compressed(f,
data=self.__data,
rowlabels=self.__rowlabels,
columnlabels=self.__columnlabels) Example 49
def save_scores(model_options,name_scores):
if not os.path.exists('scores/'):
os.system('mkdir scores/')
save_name = 'scores/scores_'+model_options['name'].split('/')[-1]
print 'Dumping scores to: '+save_name
if not os.path.isdir('scores/'):
os.mkdir('scores')
np.savez_compressed(save_name,name_scores) Example 50
def savez_compressed(file, *args, **kwds):
"""Saves one or more arrays into a file in compressed ``.npz`` format.
It is equivalent to :func:`cupy.savez` function except the output file is
compressed.
.. seealso::
:func:`cupy.savez` for more detail,
:func:`numpy.savez_compressed`
"""
args = map(cupy.asnumpy, args)
for key in kwds:
kwds[key] = cupy.asnumpy(kwds[key])
numpy.savez_compressed(file, *args, **kwds) 