diffpy.srrietveld.addon.datafile

1 ############################################################################## 2 # 3 # diffpy.srrietveld by DANSE Diffraction group 4 # Simon J. L. Billinge 5 # (c) 2009 Trustees of the Columbia University 6 # in the City of New York. All rights reserved. 7 # 8 # File coded by: Peng Tian 9 # 10 # See AUTHORS.txt for a list of people who contributed. 11 # See LICENSE.txt for license information. 12 # 13 ############################################################################## 14 15 """ datafile.py is to convert different format files into Fullprof/GSAS readable 16 format. And save x, y, sigma into data dictionary. 17 Fullprof format is the standard XYSigma format. 18 GSAS format is one of the following: 19 1. Timemap; 2. FXYE; 3. ESD; 4. STD. 20 21 """ 22 23 __id__ = "$Id: datafile.py 6420 2011-03-29 10:32:46Z yshang $" 24 25 import os 26 import math 27 from diffpy.srrietveld.utility import checkFormat 28

29 -class DataFileConverter:

30 """ Unify to convert input data file to engine file and 31 save the XYSigma into data dictionary. """ 32

33 - def __init__(self, datafullpathname, instrumentfile=None, banklist=None):

34 """ Initialization """ 35 self.inputfullname = datafullpathname 36 self.inputformat = checkFormat(datafullpathname) 37 self.instrumentfile = instrumentfile 38 self.banklist = banklist 39 40 self.datadict = {} 41 if banklist == None: 42 banklist = [1] 43 for bankid in banklist: 44 self.datadict[int(bankid)] = {"Name":"", "x":[], "y":[], "dy":[]} 45 return

46

47 - def readDataFile(self, filename):

48 """ Read input data file """ 49 inputfile = open(filename, "r") 50 filelines = inputfile.readlines() 51 inputfile.close() 52 return filelines

53

54 - def writeDataFile(self, outlines, outfilename):

55 """ Write output data file """ 56 outfile = open(outfilename, "w") 57 outfile.write(outlines) 58 outfile.close() 59 return

60

61 - def toFPFile(self, inputfile, outputfile, bankid, expfilename=""):

62 """ Convert input file to FullProf readable file. """ 63 64 dataformat = checkFormat(inputfile) 65 if dataformat == "xye": 66 self.xyeToFP(inputfile, outputfile, bankid) 67 elif dataformat == "chi": 68 self.chiToFP(inputfile, outputfile, bankid) 69 elif dataformat in ["gsa", "gda", "raw", "fxye"]: 70 outbasename = os.path.basename(outputfile) 71 processdir = outputfile.split(outbasename)[0] 72 #outrootname = os.path.splitext(outbasename)[0] 73 #outtxtname = outputfile.split(outbasename)[0] + outrootname + ".txt" 74 gsadatafile = self.exportGSASToXYE(inputfile, self.instrumentfile, 75 bankid, expfilename, processdir) 76 #self.normalizeGSAS(gsadatafile) 77 self.xyeToFP(gsadatafile, outputfile, bankid) 78 #os.remove(gsadatafile) 79 else: 80 raise NotImplementedError 81 return

82

83 - def normalizeGSAS(self, gsadatafile):

84 """ Normalize GSAS data file from rawplot to XYSigma format. """ 85 # 1. read file 86 ifile = open(gsadatafile, "r") 87 lines = ifile.readlines() 88 ifile.close() 89 90 # 2. writeout xyefile 91 outstring = "" 92 for id in range(6): 93 outstring += "#\n" 94 for line in lines[1:]: 95 terms = line.strip().split() 96 x = float(terms[0])*1000 97 y = float(terms[1])/float(terms[2]) 98 dy = math.sqrt(float(terms[1]))/float(terms[2]) 99 outstring += "%-10.4f %-10.4f %-10.5f\n" % (x, y, dy) 100 101 # 3. write out xysigma file 102 self.writeDataFile(outstring, gsadatafile) 103 return

104

105 - def toGSASFile(self, inputfile, outputfile, bankid, stype=None):

106 """ Convert input file to GSAS readable file. """ 107 108 dataformat = checkFormat(inputfile) 109 if dataformat in ["dat", "chi", "txt", "xye"]: 110 filelines = self.readDataFile(inputfile) 111 else: 112 raise NotImplementedError 113 114 datadict = self.datadict[bankid] 115 datadict["Name"] = outputfile 116 for line in filelines[4:]: 117 terms = line.strip().split() 118 if terms[0] == "#" or len(terms) != 2: 119 continue 120 else: 121 try: 122 x = float(terms[0]) 123 y = float(terms[1]) 124 # save data 125 datadict["x"].append(x) 126 except ValueError: 127 pass 128 try: 129 dy = float(terms[2]) 130 except: 131 dy = math.sqrt(abs(y)) 132 datadict["dy"].append(dy) 133 134 if stype: 135 import random 136 y_rad = random.gauss(y, dy) 137 if y_rad <= 0: 138 y = 2 * y - y_rad 139 else: 140 y = y_rad 141 datadict["y"].append(y) 142 143 outlines = self.writeGSASStr(bankid) 144 self.writeDataFile(outlines, outputfile) 145 146 return

147

148 - def readGSASFile(self, gsasfile, bankid):

149 """ Read GSAS file into datadict """ 150 # 0. convert GSAS file to XYSigma format 151 outbasename = os.path.basename(gsasfile) 152 outrootname = os.path.splitext(outbasename)[0] 153 outtxtname = gsasfile.split(outbasename)[0] + outrootname + ".txt" 154 gsadatafile = self.exportGSASToXY(gsasfile, self.instrumentfile, 155 bankid, outtxtname) 156 self.normalizeGSAS(gsadatafile) 157 # 1. read input file 158 filelines = self.readDataFile(gsadatafile) 159 160 # 2. split out the x, y, dy 161 datadict = self.datadict[bankid] 162 datadict["Name"] = gsasfile 163 for line in filelines: 164 terms = line.strip().split() 165 if terms == []: 166 continue 167 if terms[0] == "#" and filelines.index(line) < 6: 168 pass 169 else: 170 try: 171 x = float(terms[0]) 172 y = float(terms[1]) 173 dy = float(terms[2]) 174 # save data 175 datadict["x"].append(x) 176 datadict["y"].append(y) 177 datadict["dy"].append(dy) 178 except ValueError: 179 pass 180 return

181

182 - def writeGSASStr(self, bankid):

183 """ Return string of integrated intensities in GSAS format. 184 """ 185 lines = [] 186 ltitle = 'Angular Profile: %s' % os.path.basename(self.inputfullname) 187 if len(ltitle) > 80: ltitle = ltitle[:80] 188 lines.append("%-80s" % ltitle) 189 datadict = self.datadict[bankid] 190 nchan = len(datadict["y"]) 191 nrec = int(math.ceil(nchan/10.0)) 192 # two-theta0 and dtwo-theta in centidegrees 193 tth0_cdg = datadict["x"][0] * 100 194 dtth_cdg = (datadict["x"][-1] - datadict["x"][0]) / \ 195 (len(datadict["x"]) - 1) * 100 196 lbank = "BANK %4i %4i %4i CONST %9.5f %9.5f %9.5f %9.5f STD" % \ 197 (bankid, nchan, nrec, tth0_cdg, dtth_cdg, 0, 0) 198 lines.append("%-80s" % lbank) 199 lrecs = [ "%2i%6.0f" % (1, iobs) for iobs in datadict["y"] ] 200 for i in range(0, len(lrecs), 10): 201 lines.append("".join(lrecs[i:i+10])) 202 lines[-1] = "%-80s" % lines[-1] 203 rv = "\r\n".join(lines) + "\r\n" 204 return rv

205

206 - def chiToFP(self, chifile, outputfile, bankid, stype=None):

207 """ Convert .chi file to Fullprof format file """ 208 209 # 1. read input file 210 filelines = self.readDataFile(chifile) 211 212 # 2. split out the x, y, dy 213 datadict = self.datadict[bankid] 214 datadict["Name"] = outputfile 215 outlines = "#\n" 216 outlines += "#\n" 217 for line in filelines: 218 terms = line.strip().split() 219 if terms == []: 220 continue 221 if terms[0] == "#" or filelines.index(line) < 4: 222 outlines += line 223 else: 224 try: 225 x = float(terms[0]) 226 y = float(terms[1]) 227 dy = math.sqrt(y) 228 if stype: 229 import random 230 y_rad = random.gauss(y, dy) 231 if y_rad <= 0: 232 y = 2 * y - y_rad 233 else: 234 y = y_rad 235 # save data 236 datadict["x"].append(x) 237 datadict["y"].append(y) 238 datadict["dy"].append(dy) 239 # add into outlines 240 outlines += "%-15s %-15s %-15s\n"% (x, y, dy) 241 except ValueError: 242 pass 243 244 # 3. write out the Fullprof file 245 self.writeDataFile(outlines, outputfile) 246 return

247

248 - def xyeToFP(self, xyefile, outputfile, bankid, stype=None):

249 """ Convert XYSigma format file to Fullprof format file """ 250 251 # 1. read input file 252 filelines = self.readDataFile(xyefile) 253 254 # 2. split out the x, y, dy 255 datadict = self.datadict[bankid] 256 datadict["Name"] = outputfile 257 outlines = "" 258 for line in filelines: 259 terms = line.strip().split() 260 if terms == []: 261 continue 262 if terms[0] == "#" and filelines.index(line) < 6: 263 outlines += line 264 else: 265 try: 266 x = float(terms[0]) 267 y = float(terms[1]) 268 dy = float(terms[2]) 269 if stype: 270 import random 271 y_rad = random.gauss(y, dy) 272 if y_rad <= 0: 273 y = 2 * y - y_rad 274 else: 275 y = y_rad 276 # save data 277 datadict["x"].append(x) 278 datadict["y"].append(y) 279 datadict["dy"].append(dy) 280 # add into outlines 281 outlines += "%-15s %-15s %-15s\n"% (x, y, dy) 282 except ValueError: 283 pass 284 285 # 3. write out the Fullprof file 286 self.writeDataFile(outlines, outputfile) 287 return

288

289 - def xyeToChi(self, inputfile, outputfile, bankid):

290 """ Convert XYSigma format file to .chi file """ 291 292 # 1. read input file 293 filelines = self.readDataFile(inputfile) 294 295 # 2. split out the x, y, dy 296 datadict = self.datadict[bankid] 297 datadict["Name"] = outputfile 298 outlines = "" 299 for line in filelines: 300 terms = line.strip().split() 301 if terms[0] == "#" and filelines.index(line) < 4: 302 outlines += line 303 continue 304 else: 305 try: 306 x = float(terms[0]) 307 y = float(terms[1]) 308 dy = float(terms[2]) 309 # save data 310 datadict["x"].append(x) 311 datadict["y"].append(y) 312 datadict["dy"].append(dy) 313 # add into outlines 314 outlines += "%-15s %-15s %-15s\n"% (x, y, dy) 315 except ValueError: 316 pass 317 318 # 3. write out the Chi file 319 self.writeDataFile(outlines, outputfile) 320 return

321

322 - def checkEXPFile(self, expfilename, gsafbasename):

323 """Check whether the file name is right in .EXP file 324 """ 325 print expfilename, gsafbasename 326 flines = self.readDataFile(expfilename) 327 lid = -1 328 for line in flines: 329 lid += 1 330 if line.count("HST 1 HFIL") == 1: 331 terms = line.strip().split() 332 if terms[3] == gsafbasename: 333 return 334 else: 335 relen = len(gsafbasename) 336 flines[lid] = line[:14] + gsafbasename + line[relen+14:] 337 ofile = open(expfilename, "w") 338 ofile.writelines(flines) 339 ofile.close() 340 return 341 return

342

343 - def exportGSASToXYE(self, gsafname, iparmfname, bankid, expfilename, processdir):

344 """ Run Powpref and Genles to export diffraction pattern from raw GSAS file 345 """ 346 import shutil 347 import diffpy.pygsas.genles as GL 348 import diffpy.pygsas.rungsas as RG 349 350 351 expbasename = os.path.basename(expfilename) 352 exprootname = os.path.splitext(expbasename)[0] 353 expdes = os.path.join(processdir, exprootname.upper() + ".EXP") 354 shutil.copy(expfilename, expdes) 355 # check the EXP file, replace the file name if neccessary 356 gsafbasename = os.path.basename(gsafname) 357 self.checkEXPFile(expdes, gsafbasename) 358 359 desgsaf = os.path.join(processdir, gsafbasename) 360 try: 361 shutil.copy(gsafname, desgsaf) 362 except: 363 pass 364 if iparmfname: 365 iparmfbasename = os.path.basename(iparmfname) 366 desiparm = os.path.join(processdir, iparmfbasename) 367 shutil.copy(iparmfname, desiparm) 368 GL.runPowGen(exprootname, processdir, "Refine", 0, "l") 369 lstfile = os.path.join(processdir, exprootname.upper() + ".LST") 370 lstbkfile = lstfile + "bk" 371 shutil.move(lstfile, lstbkfile) 372 RG.runHstdmp(exprootname, processdir, bankid) 373 374 # write out XYE file in processdir 375 outfilename = os.path.join(processdir, exprootname + ".dat") 376 outlines = ["#\n", "#\n", "#\n", "#\n", "#\n", "#\n"] 377 hstfile = open(lstfile) 378 hstlines = hstfile.readlines() 379 hstfile.close() 380 hstlen = len(hstlines) 381 for line_id in range(1, hstlen): 382 terms = hstlines[line_id].strip().split() 383 try: 384 int(terms[0]) 385 x = float(terms[2])*1000 386 y = float(terms[3]) 387 dy = math.sqrt(1/float(terms[7])) 388 newline = "%-10.4f %-10.4f %-10.5f\n" % (x, y, dy) 389 outlines.append(newline) 390 except: 391 continue 392 ofile = open(outfilename, "w") 393 ofile.writelines(outlines) 394 ofile.close() 395 return outfilename

396

397 -def createDataDict(datainfofile, datadirectory):

398 """ Create datadict template for refine.py """ 399 datadict = {} 400 401 # 1. read datainfofile 402 infofile = open(datainfofile, "r") 403 infolines = infofile.readlines() 404 infofile.close() 405 406 # 2. split line and create datafile path 407 for line in infolines: 408 terms = line.strip().split() 409 try: 410 key = int(terms[1]) 411 except: 412 key = float(terms[1]) 413 try: 414 datadict[key] = {"Name": datadirectory + "/" + terms[0]} 415 except IOError: 416 raise IOError("%s isn't recognized!" % datainfofile) 417 418 return datadict

419

420 -def readFileList(datafilelist, enginename):

421 """ Read datafile content based on datafilelist.""" 422 import os, numpy 423 xlist = [] 424 ylist = [] 425 outfilelist = [] 426 427 m = -1 428 for datafile in datafilelist: 429 fileid = datafilelist.index(datafile) 430 dfv = DataFileConverter(datafile, banklist=[1]) 431 basefilename = os.path.basename(datafile) 432 if enginename == "fullprof": 433 outfilename = basefilename.split(".")[0] + ".dat" 434 dfv.toFPFile(datafile, outfilename, 1) 435 else: 436 outfilename = basefilename.split(".")[0] + ".gsa" 437 dfv.toGSASFile(datafile, outfilename, 1) 438 xlist.append(dfv.datadict[1]['x']) 439 ylist.append(dfv.datadict[1]['y']) 440 outfilelist.append(outfilename) 441 442 if m == -1: 443 # the first assignment 444 m = len(xlist[-1]) 445 446 # check it with the latest added x and y 447 if m != len(ylist[-1]) or m != len(xlist[-1]): 448 raise ValueError('Data is of wrong length.') 449 450 451 n = len(xlist) 452 x = numpy.empty((n, m)) 453 y = numpy.empty((n, m)) 454 455 x[:] = xlist 456 y[:] = ylist 457 458 return x, y, outfilelist

459

Source Code for Module diffpy.srrietveld.addon.datafile