Main Page | Modules | Namespace List | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Namespace Members | Class Members | File Members

hdfio2.cpp

Go to the documentation of this file.
00001 
00005 /*
00006  * Author: Steven Ludtke, 04/10/2003 (sludtke@bcm.edu)
00007  * Copyright (c) 2000-2006 Baylor College of Medicine
00008  *
00009  * This software is issued under a joint BSD/GNU license. You may use the
00010  * source code in this file under either license. However, note that the
00011  * complete EMAN2 and SPARX software packages have some GPL dependencies,
00012  * so you are responsible for compliance with the licenses of these packages
00013  * if you opt to use BSD licensing. The warranty disclaimer below holds
00014  * in either instance.
00015  *
00016  * This complete copyright notice must be included in any revised version of the
00017  * source code. Additional authorship citations may be added, but existing
00018  * author citations must be preserved.
00019  *
00020  * This program is free software; you can redistribute it and/or modify
00021  * it under the terms of the GNU General Public License as published by
00022  * the Free Software Foundation; either version 2 of the License, or
00023  * (at your option) any later version.
00024  *
00025  * This program is distributed in the hope that it will be useful,
00026  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00027  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
00028  * GNU General Public License for more details.
00029  *
00030  * You should have received a copy of the GNU General Public License
00031  * along with this program; if not, write to the Free Software
00032  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
00033  *
00034  * */
00035 
00036 #ifdef EM_HDF5
00037 
00038 //#define DEBUGHDF      1
00039 
00040 #include "hdfio2.h"
00041 #include "geometry.h"
00042 #include "ctf.h"
00043 #include "emassert.h"
00044 #include "transform.h"
00045 #include "ctf.h"
00046 #include <iostream>
00047 #include <cstring>
00048 
00049 #ifndef WIN32
00050         #include <sys/param.h>
00051 #else
00052         #define  MAXPATHLEN (MAX_PATH * 4)
00053 #endif  //WIN32
00054 
00055 using namespace EMAN;
00056 
00057 static const int ATTR_NAME_LEN = 128;
00058 
00059 HdfIO2::HdfIO2(const string & hdf_filename, IOMode rw)
00060 :       nx(1), ny(1), nz(1), is_exist(false),
00061         file(-1), group(-1), filename(hdf_filename),
00062         rw_mode(rw), initialized(false)
00063 {
00064         accprop=H5Pcreate(H5P_FILE_ACCESS);
00065 
00066         //STDIO file driver has 2G size limit on 32 bit Linux system
00067         H5Pset_fapl_sec2( accprop );
00068         //H5Pset_fapl_stdio( accprop );
00069 
00070 //      H5Pset_fapl_core( accprop, 1048576, 0  );
00071 //      H5Pset_cache(accprop)
00072         hsize_t dims=1;
00073         simple_space=H5Screate_simple(1,&dims,NULL);
00074 }
00075 
00076 HdfIO2::~HdfIO2()
00077 {
00078         H5Sclose(simple_space);
00079         H5Pclose(accprop);
00080     if (group >= 0) {
00081         H5Gclose(group);
00082     }
00083     if (file >= 0) {
00084                 H5Fflush(file,H5F_SCOPE_GLOBAL);        // If there were no resource leaks, this wouldn't be necessary...
00085                 H5Fclose(file);
00086     }
00087 #ifdef DEBUGHDF
00088         printf("HDf close\n");
00089 #endif
00090 }
00091 
00092 // This reads an already opened attribute and returns the results as an EMObject
00093 // The attribute is not closed
00094 EMObject HdfIO2::read_attr(hid_t attr) {
00095         hid_t type = H5Aget_type(attr);
00096         hid_t spc = H5Aget_space(attr);
00097         H5T_class_t cls = H5Tget_class(type);
00098         size_t sz = H5Tget_size(type);                                          // storage size, arrays handled in the 'space'
00099         hssize_t pts = H5Sget_simple_extent_npoints(spc);       // number of points > 1 if an array of floats or integers
00100 
00101         EMObject ret(0);
00102         char c;
00103         int i;
00104 //      unsigned int ui;
00105         float f,*fa;
00106         int * ia;
00107 //      unsigned int * uia;
00108         double d;
00109         char *s;
00110         vector <float> fv((size_t)pts);
00111         vector <int> iv((size_t)pts);
00112 //      vector <unsigned int> uiv(pts);
00113 
00114         float *matrix;
00115         Transform* t;
00116         Ctf* ctf;
00117 //      int r, c, k=0;
00118 
00119         switch (cls) {
00120         case H5T_INTEGER:
00121                 if(sz==1) {
00122                         H5Aread(attr,H5T_NATIVE_CHAR,&c);
00123                         bool b = false;
00124                         if(c=='T') {
00125                                 b = true;
00126                         }
00127                         else if(c=='F') {
00128                                 b = false;
00129                         }
00130                         ret = EMObject(b);
00131                 }
00132                 else if(sz==4) {
00133                         if(pts==1) {
00134                                 H5Aread(attr,H5T_NATIVE_INT,&i);
00135                                 ret=EMObject(i);
00136                         }
00137                         else {
00138                                 ia=(int *)malloc((size_t)pts*sizeof(int));
00139                                 H5Aread(attr,H5T_NATIVE_INT,ia);
00140                                 for (i=0; i<pts; i++) iv[i]=ia[i];
00141                                 free(ia);
00142                                 ret=EMObject(iv);
00143                         }
00144                 }
00145                 break;
00146 //      case H5T_UNSIGNED_INTEGER:
00147 //              if(pts==1) {
00148 //                      H5Aread(attr,H5T_NATIVE_UINT,&ui);
00149 //                      ret=EMObject(ui);
00150 //              }
00151 //              else {
00152 //                      uia=(unsigned int *)malloc(pts*sizeof(unsigned int));
00153 //                      H5Aread(attr,H5T_NATIVE_UINT,uia);
00154 //                      for (i=0; i<pts; i++) uiv[i]=uia[i];
00155 //                      free(uia);
00156 //                      ret=EMObject(uiv);
00157 //              }
00158 //              break;
00159         case H5T_FLOAT:
00160                 if (sz==4) {
00161                         if (pts==1) {
00162                                 H5Aread(attr,H5T_NATIVE_FLOAT,&f);
00163                                 ret=EMObject(f);
00164                         }
00165                         else {
00166                                 fa=(float *)malloc((size_t)pts*sizeof(float));
00167                                 H5Aread(attr,H5T_NATIVE_FLOAT,fa);
00168                                 for (i=0; i<pts; i++) fv[i]=fa[i];
00169                                 free(fa);
00170                                 ret=EMObject(fv);
00171                         }
00172                 }
00173                 else if (sz==8) {
00174                         H5Aread(attr,H5T_NATIVE_DOUBLE,&d);
00175                         ret=EMObject(d);
00176                 }
00177                 break;
00178         case H5T_STRING:
00179                 s=(char *)malloc(sz+1);
00180                 H5Aread(attr,type,s);
00181 //              H5Aread(attr,H5T_NATIVE_CHAR,s);
00182                 if(s[0] == 'O' && isdigit(s[1])) {
00183                         ctf = new EMAN1Ctf();
00184                         ctf->from_string(string(s));
00185                         ret = EMObject(ctf);
00186                         delete ctf;
00187                 }
00188                 else if(s[0] == 'E' && isdigit(s[1])) {
00189                         ctf = new EMAN2Ctf();
00190                         ctf->from_string(string(s));
00191                         ret = EMObject(ctf);
00192                         delete ctf;
00193                 }
00194                 else {
00195                         ret=EMObject(s);
00196                 }
00197                 free(s);
00198                 break;
00199         case H5T_COMPOUND:
00200                 matrix = (float*)malloc(12*sizeof(float));
00201                 H5Aread(attr, type, matrix);
00202 //              ret.create_transform3d_by_array(trans3d);
00203                 t = new Transform(matrix);
00204                 ret = EMObject(t);
00205                 free(matrix);
00206                 delete t; t=0;
00207 
00208 //              trans3d = (float*)malloc(16*sizeof(float));     //16 float for a Transform3D object
00209 //              H5Aread(attr, type, trans3d);
00210 // //           ret.create_transform3d_by_array(trans3d);
00211 //              trans = new Transform3D();
00212 //              for(r=0; r<4; ++r) {
00213 //                      for(c=0; c<4; ++c) {
00214 //                              trans->set(r, c, trans3d[k]);
00215 //                              ++k;
00216 //                      }
00217 //              }
00218 //              ret = EMObject(trans);
00219 //              free(trans3d);
00220                 break;
00221         default:
00222                 LOGERR("Unhandled HDF5 metadata %d", cls);
00223         }
00224 
00225         H5Sclose(spc);
00226         H5Tclose(type);
00227 
00228         return ret;
00229 }
00230 
00231 // This writes an attribute with specified name to a given open object
00232 // The attribute is opened and closed. returns 0 on success
00233 int HdfIO2::write_attr(hid_t loc,const char *name,EMObject obj) {
00234         hid_t type=0;
00235         hid_t spc=0;
00236         hsize_t dims=1;
00237         vector <float> fv;
00238         vector <int> iv;
00239         switch(obj.get_type())
00240         {
00241         case EMObject::BOOL:
00242                 type=H5Tcopy(H5T_NATIVE_CHAR);
00243                 spc=H5Scopy(simple_space);
00244                 break;
00245         case EMObject::INT:
00246                 type=H5Tcopy(H5T_NATIVE_INT);
00247                 spc=H5Scopy(simple_space);
00248                 break;
00249         case EMObject::UNSIGNEDINT:
00250                 type=H5Tcopy(H5T_NATIVE_UINT);
00251                 spc=H5Scopy(simple_space);
00252                 break;
00253         case EMObject::FLOAT:
00254                 type=H5Tcopy(H5T_NATIVE_FLOAT);
00255                 spc=H5Scopy(simple_space);
00256                 break;
00257         case EMObject::DOUBLE:
00258                 type=H5Tcopy(H5T_NATIVE_DOUBLE);
00259                 spc=H5Scopy(simple_space);
00260                 break;
00261         case EMObject::STRING:
00262         case EMObject::CTF:
00263                 type=H5Tcopy(H5T_C_S1);
00264                 H5Tset_size(type,strlen((const char *)obj)+1);
00265                 spc=H5Screate(H5S_SCALAR);
00266                 break;
00267         case EMObject::FLOATARRAY:
00268                 type=H5Tcopy(H5T_NATIVE_FLOAT);
00269                 fv=obj;
00270                 dims=fv.size();
00271                 spc=H5Screate_simple(1,&dims,NULL);
00272                 break;
00273         case EMObject::INTARRAY:
00274                 type=H5Tcopy(H5T_NATIVE_INT);
00275                 iv=obj;
00276                 dims=iv.size();
00277                 spc=H5Screate_simple(1,&dims,NULL);
00278                 break;
00279         case EMObject::TRANSFORM:
00280                 type = H5Tcreate(H5T_COMPOUND, 12 * sizeof(float)); //Transform is a 3x4 matrix
00281                 H5Tinsert(type, "00", 0, H5T_NATIVE_FLOAT);
00282                 H5Tinsert(type, "01", 1*sizeof(float), H5T_NATIVE_FLOAT);
00283                 H5Tinsert(type, "02", 2*sizeof(float), H5T_NATIVE_FLOAT);
00284                 H5Tinsert(type, "03", 3*sizeof(float), H5T_NATIVE_FLOAT);
00285                 H5Tinsert(type, "10", 4*sizeof(float), H5T_NATIVE_FLOAT);
00286                 H5Tinsert(type, "11", 5*sizeof(float), H5T_NATIVE_FLOAT);
00287                 H5Tinsert(type, "12", 6*sizeof(float), H5T_NATIVE_FLOAT);
00288                 H5Tinsert(type, "13", 7*sizeof(float), H5T_NATIVE_FLOAT);
00289                 H5Tinsert(type, "20", 8*sizeof(float), H5T_NATIVE_FLOAT);
00290                 H5Tinsert(type, "21", 9*sizeof(float), H5T_NATIVE_FLOAT);
00291                 H5Tinsert(type, "22", 10*sizeof(float), H5T_NATIVE_FLOAT);
00292                 H5Tinsert(type, "23", 11*sizeof(float), H5T_NATIVE_FLOAT);
00293                 H5Tpack(type);
00294 
00295                 dims = 1;       //one compound type
00296                 spc = H5Screate_simple(1, &dims, NULL);
00297                 break;
00298         case EMObject::STRINGARRAY:
00299         case EMObject::EMDATA:
00300         case EMObject::XYDATA:
00301         case EMObject::FLOAT_POINTER:
00302         case EMObject::INT_POINTER:
00303         case EMObject::VOID_POINTER:
00304                 return -1;
00305                 break;
00306         case EMObject::UNKNOWN:
00307                 break;
00308         }
00309 
00310     //we need this delete attribute call here, even we called erase_header()
00311     //at the biginning of write_header(), since the  "imageid_max" need be updated correctly.
00312         if( H5Adelete(loc,name) < 0 ) {
00313 #ifdef DEBUGHDF
00314                 LOGERR("Attribute %s deletion error in write_attr().\n", name);
00315 #endif
00316         }
00317         else {
00318 #ifdef DEBUGHDF
00319                 printf("delete attribute %s successfully in write_attr().\n", name);
00320 #endif
00321         }
00322         hid_t attr = H5Acreate(loc,name,type,spc,H5P_DEFAULT);
00323 
00324         bool b;
00325         char c;
00326         int i;
00327         float f,*fa;
00328         int * ia;
00329         unsigned int ui;
00330         double d;
00331         const char *s;
00332         Transform * tp;
00333         switch(obj.get_type()) {
00334         case EMObject::BOOL:
00335                 b = (bool)obj;
00336                 if(b) {
00337                         c = 'T';
00338                 } else {
00339                         c = 'F';
00340                 }
00341                 H5Awrite(attr,type,&c);
00342                 break;
00343         case EMObject::INT:
00344                 i=(int)obj;
00345                 H5Awrite(attr,type,&i);
00346                 break;
00347         case EMObject::UNSIGNEDINT:
00348                 ui=(unsigned int)obj;
00349                 H5Awrite(attr,type,&ui);
00350                 break;
00351         case EMObject::FLOAT:
00352                 f=(float)obj;
00353                 H5Awrite(attr,type,&f);
00354                 break;
00355         case EMObject::DOUBLE:
00356                 d=(double)obj;
00357                 H5Awrite(attr,type,&d);
00358                 break;
00359         case EMObject::STRING:
00360         case EMObject::CTF:
00361                 s=(const char *)obj;
00362                 H5Awrite(attr,type,s);
00363                 break;
00364         case EMObject::FLOATARRAY:
00365                 fa=(float *)malloc(fv.size()*sizeof(float));
00366                 for (ui=0; ui<fv.size(); ui++) fa[ui]=fv[ui];
00367                 H5Awrite(attr,type,fa);
00368                 free(fa);
00369                 break;
00370         case EMObject::INTARRAY:
00371                 ia=(int *)malloc(iv.size()*sizeof(int));
00372                 for (ui=0; ui<iv.size(); ui++) ia[ui]=iv[ui];
00373                 H5Awrite(attr,type,ia);
00374                 free(ia);
00375                 break;
00376         case EMObject::TRANSFORM:
00377         {
00378                 tp = (Transform *)obj;
00379                 fa = (float *)malloc(12*sizeof(float));
00380                 int r, c, k=0;
00381                 for(r=0; r<3; ++r) {
00382                         for(c=0; c<4; ++c) {
00383                                 fa[k] = tp->at(r,c);
00384                                 ++k;
00385                         }
00386                 }
00387                 H5Awrite(attr,type,fa);
00388                 free(fa);
00389         }
00390                 break;
00391 //      case EMObject::STRINGARRAY:
00392 //      case EMObject::EMDATA:
00393 //      case EMObject::XYDATA:
00394 //              return -1;
00395 //              break;
00396         default:
00397                 LOGERR("Unhandled HDF5 metadata '%s'", name);
00398 
00399         }
00400 
00401         herr_t ret1 = H5Tclose(type);
00402         herr_t ret2 = H5Sclose(spc);
00403         herr_t ret3 = H5Aclose(attr);
00404         if(ret1>=0 && ret2>=0 && ret3>=0) {
00405                 return 0;
00406         }
00407         else {
00408                 LOGERR("close error in write_attr()\n");
00409                 return -1;
00410         }
00411 }
00412 
00413 // Initializes the file for read-only or read-write access
00414 // Data is stored under /MDF/images
00415 // An attribute named imageid_max stores the number of the highest
00416 // numbered image in the file.
00417 // A group is then made for each individual image, all metadata for the
00418 // individual images is currently associated with the GROUP, not the dataset
00419 // dataset-specific data could also be associated with the dataset in
00420 // future. At the moment, there is only a single dataset in each group.
00421 void HdfIO2::init()
00422 {
00423         ENTERFUNC;
00424         if (initialized) {
00425                 return;
00426         }
00427 #ifdef DEBUGHDF
00428         printf("init\n");
00429 #endif
00430 
00431         H5Eset_auto(0, 0);      // Turn off console error logging.
00432 
00433         if (rw_mode == READ_ONLY) {
00434                 file = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, accprop);
00435                 if (file<0) throw FileAccessException(filename);
00436         }
00437         else {
00438                 file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, accprop);
00439                 if (file < 0) {
00440                         file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, accprop);
00441                         if (file < 0) {
00442                                 throw FileAccessException(filename);
00443                         }
00444                         else {
00445 #ifdef DEBUGHDF
00446                                 printf("File truncated or new file created\n");
00447 #endif
00448                         }
00449                 }
00450         }
00451 
00452         group=H5Gopen(file,"/MDF/images");
00453         if (group<0) {
00454                 if (rw_mode == READ_ONLY) throw ImageReadException(filename,"HDF5 file has no image data (no /MDF group)");
00455                 group=H5Gcreate(file,"/MDF",64);                // create the group for Macromolecular data
00456                 if (group<0) throw ImageWriteException(filename,"Unable to add image group (/MDF) to HDF5 file");
00457                 H5Gclose(group);
00458                 group=H5Gcreate(file,"/MDF/images",4096);               // create the group for images/volumes
00459                 if (group<0) throw ImageWriteException(filename,"Unable to add image group (/MDF/images) to HDF5 file");
00460                 write_attr(group,"imageid_max",EMObject(-1));
00461         }
00462         initialized = true;
00463         EXITFUNC;
00464 }
00465 
00466 
00467 // If this version of init() returns -1, then we have an old-style HDF5 file
00468 int HdfIO2::init_test()
00469 {
00470         ENTERFUNC;
00471         if (initialized) {
00472                 return 1;
00473         }
00474 #ifdef DEBUGHDF
00475         printf("init_test\n");
00476 #endif
00477 
00478         H5Eset_auto(0, 0);      // Turn off console error logging.
00479 
00480         hid_t fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5Pcreate(H5P_FILE_ACCESS));
00481         hid_t groupid = H5Gopen(fileid, "/");
00482         hid_t attid = H5Aopen_name(groupid, "num_dataset");
00483 
00484         if (attid < 0) {
00485                 H5Gclose(groupid);
00486                 H5Fclose(fileid);
00487                 init();
00488                 EXITFUNC;
00489                 return 0;
00490         }
00491         else {
00492                 H5Aclose(attid);
00493                 H5Gclose(groupid);
00494                 H5Fclose(fileid);
00495                 EXITFUNC;
00496                 return -1;
00497         }
00498 }
00499 
00500 bool HdfIO2::is_valid(const void *first_block)
00501 {
00502         ENTERFUNC;
00503 
00504         if (first_block) {
00505                 char signature[8] = { 137,72,68,70,13,10,26,10 };
00506                 if (strncmp((const char *)first_block,signature,8)==0) return true;
00507                 // const char* f=(const char *)first_block;
00508                 // printf("bad hdf signature %d %d %d %d %d %d %d %d",f[0],f[1],f[2],f[3],f[4],f[5],f[6],f[7]);
00509                 return false;
00510         }
00511         EXITFUNC;
00512         return false;
00513 }
00514 
00515 // Reads all of the attributes from the /MDF/images/<imgno> group
00516 int HdfIO2::read_header(Dict & dict, int image_index, const Region * area, bool)
00517 {
00518         ENTERFUNC;
00519         init();
00520 #ifdef DEBUGHDF
00521         printf("read_head %d\n", image_index);
00522 #endif
00523         int i;
00524         // Each image is in a group for later expansion. Open the group
00525         char ipath[50];
00526         sprintf(ipath,"/MDF/images/%d", image_index);
00527         hid_t igrp=H5Gopen(file, ipath);
00528 
00529         int nattr=H5Aget_num_attrs(igrp);
00530 
00531         char name[ATTR_NAME_LEN];
00532         for (i=0; i<nattr; i++) {
00533                 hid_t attr=H5Aopen_idx(igrp, i);
00534                 H5Aget_name(attr,127,name);
00535                 if (strncmp(name,"EMAN.", 5)!=0) {
00536                         H5Aclose(attr);
00537                         continue;
00538                 }
00539                 EMObject val=read_attr(attr);
00540                 dict[name+5]=val;
00541                 H5Aclose(attr);
00542         }
00543 
00544         if(dict.has_key("ctf")) {
00545                 string ctfString = (string)dict["ctf"];
00546                 if(ctfString.substr(0, 1) == "O") {
00547                         Ctf * ctf_ = new EMAN1Ctf();
00548                         ctf_->from_string(ctfString);
00549                         dict.erase("ctf");
00550                         dict["ctf"] = ctf_;
00551                         delete ctf_;
00552                 }
00553                 else if(ctfString.substr(0, 1) == "E") {
00554                         Ctf * ctf_ = new EMAN2Ctf();
00555                         ctf_->from_string(ctfString);
00556                         dict.erase("ctf");
00557                         dict["ctf"] = ctf_;
00558                         delete ctf_;
00559                 }
00560         }
00561 
00562         if(area) {
00563                 check_region(area, IntSize(dict["nx"], dict["ny"], dict["nz"]), false, false);
00564 
00565                 dict["nx"] = area->get_width();
00566                 dict["ny"] = area->get_height();
00567                 dict["nz"] = area->get_depth();
00568 
00569                 if( dict.has_key("apix_x") && dict.has_key("apix_y") && dict.has_key("apix_z") )
00570                 {
00571                         if( dict.has_key("origin_x") && dict.has_key("origin_y") && dict.has_key("origin_z") )
00572                         {
00573                                 float xorigin = dict["origin_x"];
00574                                 float yorigin = dict["origin_y"];
00575                                 float zorigin = dict["origin_z"];
00576 
00577                                 float apix_x = dict["apix_x"];
00578                                 float apix_y = dict["apix_y"];
00579                                 float apix_z = dict["apix_z"];
00580 
00581                                 dict["origin_x"] = xorigin + apix_x * area->origin[0];
00582                                 dict["origin_y"] = yorigin + apix_y * area->origin[1];
00583                                 dict["origin_z"] = zorigin + apix_z * area->origin[2];
00584                         }
00585                 }
00586         }
00587 
00588         H5Gclose(igrp);
00589 
00590         //Get the data type from data set, HDF5 file header attribute 'datatype' may be wrong
00591         sprintf(ipath,"/MDF/images/%d/image",image_index);
00592         hid_t ds=H5Dopen(file,ipath);
00593 
00594         if(ds>0) {      //ds>0 means successfully open the dataset
00595                 hid_t dt = H5Dget_type(ds);
00596 
00597                 switch(H5Tget_size(dt)) {
00598                 case 4:
00599                         dict["datatype"] = (int)EMUtil::EM_FLOAT;
00600                         break;
00601                 case 2:
00602                         dict["datatype"] = (int)EMUtil::EM_USHORT;
00603                         break;
00604                 case 1:
00605                         dict["datatype"] = (int)EMUtil::EM_UCHAR;
00606                         break;
00607                 default:
00608                         throw ImageReadException(filename, "EMAN does not support this data type.");
00609                 }
00610 
00611                 H5Tclose(dt);
00612         }
00613 
00614         H5Dclose(ds);
00615 
00616         EXITFUNC;
00617         return 0;
00618 }
00619 
00620 // This erases any existing attributes from the image group
00621 // prior to writing a new header. For a new image there
00622 // won't be any, so this should be harmless.
00623 int HdfIO2::erase_header(int image_index)
00624 {
00625         ENTERFUNC;
00626 
00627         if(image_index < 0) return 0; //image_index<0 for appending image, no need for erasing
00628 
00629         init();
00630 #ifdef DEBUGHDF
00631         printf("erase_head %d\n",image_index);
00632 #endif
00633         int i;
00634         // Each image is in a group for later expansion. Open the group
00635         char ipath[50];
00636         sprintf(ipath,"/MDF/images/%d", image_index);
00637         hid_t igrp=H5Gopen(file, ipath);
00638 
00639         int nattr=H5Aget_num_attrs(igrp);
00640 
00641         char name[ATTR_NAME_LEN];
00642         for (i=0; i<nattr; i++) {
00643                 hid_t attr = H5Aopen_idx(igrp, 0); //use 0 as index here, since the H5Adelete() will change the index
00644                 H5Aget_name(attr,127,name);
00645                 H5Aclose(attr);
00646                 if( H5Adelete(igrp,name) < 0 ) {
00647                         LOGERR("attribute %s deletion error in erase_header().\n", name);
00648                 }
00649         }
00650 
00651         H5Gclose(igrp);
00652         EXITFUNC;
00653         return 0;
00654 }
00655 
00656 
00657 int HdfIO2::read_data(float *data, int image_index, const Region *area, bool)
00658 {
00659         ENTERFUNC;
00660 #ifdef DEBUGHDF
00661         printf("read_data %d\n",image_index);
00662 #endif
00663 
00664         char ipath[50];
00665         sprintf(ipath,"/MDF/images/%d/image",image_index);
00666         hid_t ds=H5Dopen(file,ipath);
00667         if (ds<0) throw ImageWriteException(filename,"Image does not exist");
00668         hid_t spc=H5Dget_space(ds);
00669         hid_t dt = H5Dget_type(ds);
00670 
00671         hsize_t dims_out[3];
00672         hsize_t rank = H5Sget_simple_extent_ndims(spc);
00673 
00674         H5Sget_simple_extent_dims(spc, dims_out, NULL);
00675         if(rank == 1) {
00676                 nx = dims_out[0];
00677                 ny = 1;
00678                 nz = 1;
00679         }
00680         else if(rank == 2) {
00681                 nx = dims_out[1];
00682                 ny = dims_out[0];
00683                 nz = 1;
00684         }
00685         else if(rank == 3) {
00686                 nx = dims_out[2];
00687                 ny = dims_out[1];
00688                 nz = dims_out[0];
00689         }
00690 
00691         if (area) {
00692                 hid_t memoryspace = 0;
00693 
00694                 /*Get the file dataspace - the region we want to read in the file*/
00695                 int x0 = 0, y0 = 0, z0 = 0;             //the coordinates for up left corner, trim to be within image bound
00696                 int x1 = 0, y1 = 0, z1 = 0;             //the coordinates for down right corner, trim to be within image bound
00697                 int nx1 = 1, ny1 = 1, nz1 = 1;  //dimensions of the sub-region, actual region read form file
00698                 if(rank == 3) {
00699                         hsize_t     doffset[3];             /* hyperslab offset in the file */
00700                         doffset[2] = (hsize_t)(area->x_origin() < 0 ? 0 : area->x_origin());
00701                         doffset[1] = (hsize_t)(area->y_origin() < 0 ? 0 : area->y_origin());
00702                         doffset[0] = (hsize_t)(area->z_origin() < 0 ? 0 : area->z_origin());
00703                         x0 = (int)doffset[0];
00704                         y0 = (int)doffset[1];
00705                         z0 = (int)doffset[2];
00706 
00707                         z1 = (int)(area->x_origin() + area->get_width());
00708                         z1 = (int)(z1 > static_cast<int>(nx) ? nx : z1);
00709 
00710                         y1 = (int)(area->y_origin() + area->get_height());
00711                         y1 = (int)(y1 > static_cast<int>(ny) ? ny : y1);
00712                         if(y1 <= 0) {
00713                                 y1 = 1;
00714                         }
00715 
00716                         x1 = (int)(area->z_origin() + area->get_depth());
00717                         x1 = (int)(x1 > static_cast<int>(nz) ? nz : x1);
00718                         if(x1 <= 0) {
00719                                 x1 = 1;
00720                         }
00721 
00722                         if(x1 < x0 || y1< y0 || z1 < z0) return 0; //out of bounds, this is fine, nothing happens
00723 
00724                         hsize_t     dcount[3];              /* size of the hyperslab in the file */
00725                         dcount[0] = x1 - doffset[0];
00726                         dcount[1] = y1 - doffset[1];
00727                         dcount[2] = z1 - doffset[2];
00728 
00729                         H5Sselect_hyperslab (spc, H5S_SELECT_SET, (const hsize_t*)doffset, NULL, (const hsize_t*)dcount, NULL);
00730 
00731                         /*Define memory dataspace - the memory we will created for the region*/
00732                         hsize_t     dims[3];              /* size of the region in the memory */
00733                         dims[0] = dcount[2]?dcount[2]:1;
00734                         dims[1] = dcount[1]?dcount[1]:1;
00735                         dims[2] = dcount[0]?dcount[0]:1;
00736                         nx1 = (int)dims[0];
00737                         ny1 = (int)dims[1];
00738                         nz1 = (int)dims[2];
00739 
00740                         memoryspace = H5Screate_simple(3, dims, NULL);
00741                 }
00742                 else if(rank == 2) {
00743                         hsize_t     doffset[2];             /* hyperslab offset in the file */
00744                         doffset[1] = (hsize_t)(area->x_origin() < 0 ? 0 : area->x_origin());
00745                         doffset[0] = (hsize_t)(area->y_origin() < 0 ? 0 : area->y_origin());
00746                         x0 = (int)doffset[0];
00747                         y0 = (int)doffset[1];
00748                         z0 = 1;
00749 
00750                         y1 = (int)(area->x_origin() + area->get_width());
00751                         y1 = (int)(y1 > static_cast<int>(nx) ? nx : y1);
00752 
00753                         x1 = (int)(area->y_origin() + area->get_height());
00754                         x1 = (int)(x1 > static_cast<int>(ny) ? ny : x1);
00755                         if(x1 <= 0) {
00756                                 x1 = 1;
00757                         }
00758 
00759                         z1 = 1;
00760 
00761                         if(x1 < x0 || y1< y0) return 0; //out of bounds, this is fine, nothing happens
00762 
00763                         hsize_t     dcount[2];              /* size of the hyperslab in the file */
00764                         dcount[0] = x1 - doffset[0];
00765                         dcount[1] = y1 - doffset[1];
00766 
00767                         H5Sselect_none(spc);
00768                         H5Sselect_hyperslab (spc, H5S_SELECT_SET, (const hsize_t*)doffset, NULL, (const hsize_t*)dcount, NULL);
00769 
00770                         /*Define memory dataspace - the memory we will created for the region*/
00771                         hsize_t     dims[2];              /* size of the region in the memory */
00772                         dims[0] = (hsize_t)(dcount[1]?dcount[1]:1);
00773                         dims[1] = (hsize_t)(dcount[0]?dcount[0]:1);
00774                         nx1 = (int)dims[0];
00775                         ny1 = (int)dims[1];
00776                         nz1 = 1;
00777 
00778                         memoryspace = H5Screate_simple(2, dims, NULL);
00779                 }
00780 
00781                 if( (area->x_origin()>=0) && (area->y_origin()>=0) && (area->z_origin()>=0) && ((hsize_t)(area->x_origin() + area->get_width())<=nx) && ((hsize_t)(area->y_origin() + area->get_height())<=ny) && ((hsize_t)(area->z_origin() + area->get_depth())<=nz) ){      //the region is in boundary
00782                         H5Dread(ds,H5T_NATIVE_FLOAT,memoryspace,spc,H5P_DEFAULT,data);
00783                 }
00784                 else {  //the region are partial out of boundary
00785                         /* When the requested region has some part out of image boundary,
00786                          * we need read the sub-area which is within image,
00787                          * and fill the out of boundary part with zero.
00788                          * We actually read the sub-region from HDF by hyperslab I/O,
00789                          * then copy it back to the pre-allocated region.*/
00790                         float * subdata = new float[nx1*ny1*nz1];
00791 
00792 
00793                         H5Dread(ds,H5T_NATIVE_FLOAT,memoryspace,spc,H5P_DEFAULT,subdata);
00794 
00795                         int xd0=0, yd0=0, zd0=0;        //The coordinates of the top-left corner sub-region in region
00796                         size_t clipped_row_size = 0;
00797                         if(rank == 3) {
00798                                 xd0 = (int) (area->x_origin() < 0 ? -area->x_origin() : 0);
00799                                 yd0 = (int) (area->y_origin() < 0 ? -area->y_origin() : 0);
00800                                 zd0 = (int) (area->z_origin() < 0 ? -area->z_origin() : 0);
00801                                 clipped_row_size = (z1-z0)* sizeof(float);
00802                         }
00803                         else if(rank == 2) {
00804                                 xd0 = (int) (area->x_origin() < 0 ? -area->x_origin() : 0);
00805                                 yd0 = (int) (area->y_origin() < 0 ? -area->y_origin() : 0);
00806                                 clipped_row_size = (y1-y0)* sizeof(float);
00807                         }
00808 
00809                         int src_secsize = nx1 * ny1;
00810                         int dst_secsize = (int)(area->get_width())*(int)(area->get_height());
00811 
00812                         float * src = subdata;
00813                         float * dst = data + zd0*dst_secsize + yd0*(int)(area->get_width()) + xd0;
00814 
00815                         int src_gap = src_secsize - (y1-y0) * nx1;
00816                         int dst_gap = dst_secsize - (y1-y0) * (int)(area->get_width());
00817 
00818                         for(int i = 0; i<nz1; ++i) {
00819                                 for(int j = 0; j<ny1; ++j) {
00820                                         EMUtil::em_memcpy(dst, src, clipped_row_size);
00821 
00822                                         src += nx1;
00823                                         dst += (int)(area->get_width());
00824                                 }
00825                                 src += src_gap;
00826                                 dst += dst_gap;
00827                         }
00828 
00829                         delete [] subdata;
00830                 }
00831                 H5Sclose(memoryspace);
00832         } else {
00833                 hsize_t size = nx*ny*nz;
00834                 hsize_t i=0;
00835                 hsize_t j=0;
00836                 unsigned short *usdata = (unsigned short *) data;
00837                 unsigned char *cdata = (unsigned char *) data;
00838                 switch(H5Tget_size(dt)) {
00839                 case 4:
00840                         H5Dread(ds,H5T_NATIVE_FLOAT,spc,spc,H5P_DEFAULT,data);
00841                         break;
00842                 case 2:
00843                         H5Dread(ds,H5T_NATIVE_USHORT,spc,spc,H5P_DEFAULT,usdata);
00844                         for (i = 0; i < size; ++i) {
00845                                 j = size - 1 - i;
00846                                 data[j] = static_cast < float >(usdata[j]);
00847                         }
00848                         break;
00849                 case 1:
00850                         H5Dread(ds,H5T_NATIVE_UCHAR,spc,spc,H5P_DEFAULT,cdata);
00851                         for (i = 0; i < size; ++i) {
00852                                 j = size - 1 - i;
00853                                 data[j] = static_cast < float >(cdata[j]);
00854                         }
00855                         break;
00856                 default:
00857                         throw ImageReadException(filename, "EMAN does not support this data type.");
00858                 }
00859         }
00860 
00861         H5Tclose(dt);
00862         H5Sclose(spc);
00863         H5Dclose(ds);
00864         EXITFUNC;
00865         return 0;
00866 }
00867 
00868 
00869 // Writes all attributes in 'dict' to the image group
00870 // Creation of the image dataset is also handled here
00871 int HdfIO2::write_header(const Dict & dict, int image_index, const Region* area,
00872                                                 EMUtil::EMDataType dt, bool)
00873 {
00874 #ifdef DEBUGHDF
00875         printf("write_head %d\n",image_index);
00876 #endif
00877         ENTERFUNC;
00878         init();
00879 
00880         nx = (int)dict["nx"];
00881         ny = (int)dict["ny"];
00882         nz = (int)dict["nz"];
00883 
00884         if(image_index<0) {
00885                 image_index = get_nimg();
00886         }
00887 
00888         // If image_index<0 append, and make sure the max value in the file is correct
00889         // though this is normally handled by EMData.write_image()
00890         hid_t attr=H5Aopen_name(group,"imageid_max");
00891         int nimg = read_attr(attr);
00892         H5Aclose(attr);
00893 
00894         unsigned int i;
00895         if (image_index<0) image_index=nimg+1;
00896         if (image_index>nimg) {
00897                 write_attr(group,(const char *)"imageid_max",EMObject(image_index));
00898         }
00899 
00900         // Each image is in a group for later expansion. Open the group, create if necessary
00901         char ipath[50];
00902         sprintf(ipath,"/MDF/images/%d",image_index);
00903         hid_t igrp=H5Gopen(file,ipath);
00904 
00905         if (igrp<0) {   //group not existed
00906                 is_exist = false;
00907                 // Need to create a new image group
00908                 igrp=H5Gcreate(file,ipath,64);          // The image is a group, with attributes on the group
00909                 if (igrp<0) throw ImageWriteException(filename,"Unable to add /MDF/images/# to HDF5 file");
00910         }
00911         //if group already existed, erase the header and unlink the existing dataset first
00912         else {
00913                 is_exist = true;
00914                 int nattr=H5Aget_num_attrs(igrp);
00915                 char name[ATTR_NAME_LEN];
00916                 Dict dict2;
00917                 for (int i=0; i<nattr; i++) {
00918                         hid_t attr=H5Aopen_idx(igrp, i);
00919                         H5Aget_name(attr,127,name);
00920                         if (strncmp(name,"EMAN.", 5)!=0) {
00921                                 H5Aclose(attr);
00922                                 continue;
00923                         }
00924                         EMObject val=read_attr(attr);
00925                         dict2[name+5]=val;
00926                         H5Aclose(attr);
00927 
00928                         if(!dict2.has_key("datatype")) {//by default, HDF5 is written as float
00929                                 dict2["datatype"] = (int)EMUtil::EM_FLOAT;
00930                         }
00931                 }
00932 
00933                 erase_header(image_index);
00934 
00935                 //change the size or data type of a image,
00936                 //the existing data set is invalid, unlink it
00937                 if( (int)dict["nx"]*(int)dict["ny"]*(int)dict["nz"] !=
00938                         (int)dict2["nx"]*(int)dict2["ny"]*(int)dict2["nz"] ||
00939                         dict["datatype"] != dict2["datatype"] ) {
00940                         sprintf(ipath,"/MDF/images/%d/image",image_index);
00941                         H5Gunlink(igrp, ipath);
00942                 }
00943         }
00944 
00945         if(area) {
00946                 check_region(area, IntSize(dict["nx"], dict["ny"], dict["nz"]), false, true);
00947         }
00948 
00949         // Write the attributes to the group
00950         vector <string> keys=dict.keys();
00951 
00952         for (i=0; i<keys.size(); i++) {
00953                 string s("EMAN.");
00954                 s+=keys[i];
00955                 write_attr(igrp,s.c_str(),dict[keys[i]]);
00956         }
00957 
00958         H5Gclose(igrp);
00959         EXITFUNC;
00960         return 0;
00961 }
00962 
00963 // Writes the actual image data to the corresponding dataset (already created)
00964 int HdfIO2::write_data(float *data, int image_index, const Region* area,
00965                                           EMUtil::EMDataType dt, bool)
00966 {
00967         ENTERFUNC;
00968 
00969 #ifdef DEBUGHDF
00970         printf("write_data %d\n",image_index);
00971 #endif
00972 
00973         if (image_index<0) {
00974                 hid_t attr=H5Aopen_name(group,"imageid_max");
00975                 image_index = read_attr(attr);
00976                 H5Aclose(attr);
00977         }
00978 
00979         hid_t spc;      //dataspace
00980         hid_t ds;       //dataset
00981         char ipath[50];
00982         sprintf(ipath,"/MDF/images/%d/image",image_index);
00983 
00984         // Now create the actual image dataset
00985         if (nz==1)  {
00986                 hsize_t dims[2]= { ny,nx };
00987                 spc=H5Screate_simple(2,dims,NULL);
00988         }
00989         else {
00990                 hsize_t dims[3]= { nz, ny, nx };
00991                 spc=H5Screate_simple(3,dims,NULL);
00992         }
00993 
00994         ds=H5Dopen(file,ipath);
00995         if(ds<0) {//new dataset
00996                 switch(dt) {
00997                 case EMUtil::EM_FLOAT:
00998                         ds=H5Dcreate(file,ipath, H5T_NATIVE_FLOAT, spc, H5P_DEFAULT );
00999                         break;
01000                 case EMUtil::EM_USHORT:
01001                         ds=H5Dcreate(file,ipath, H5T_NATIVE_USHORT, spc, H5P_DEFAULT );
01002                         break;
01003                 case EMUtil::EM_UCHAR:
01004                         ds=H5Dcreate(file,ipath, H5T_NATIVE_UCHAR, spc, H5P_DEFAULT );
01005                         break;
01006                 default:
01007                         throw ImageWriteException(filename,"HDF5 does not support this data format");
01008                 }
01009         }
01010 
01011         //convert data to unsigned short, unsigned char...
01012         hsize_t size = nx*ny*nz;
01013         unsigned char *cdata = 0;
01014         unsigned short *usdata = 0;
01015         float rendermin = 0.0f;
01016         float rendermax = 0.0f;
01017         EMUtil::getRenderMinMax(data, nx, ny, rendermin, rendermax, nz);
01018 
01019         if(area) {
01020                 hsize_t doffset[3];             /*hyperslab offset in the file*/
01021                 doffset[0] = (hsize_t)(area->x_origin());
01022                 doffset[1] = (hsize_t)(area->y_origin());
01023                 doffset[2] = (hsize_t)(area->z_origin());
01024 
01025                 hsize_t dcount[3];              /*size of the hyperslab in the file*/
01026                 dcount[0] = (hsize_t)(area->get_width());
01027                 dcount[1] = (hsize_t)(area->get_height()?area->get_height():1);
01028                 dcount[2] = (hsize_t)(area->get_depth()?area->get_depth():1);
01029 
01030                 H5Sselect_hyperslab(spc, H5S_SELECT_SET, (const hsize_t*)doffset, NULL, (const hsize_t*)dcount, NULL);
01031 
01032                 /*Create memory space with size of the region.*/
01033                 hsize_t dims[3];        /*size of the region in the memory*/
01034                 dims[0] = (hsize_t)(area->get_width());
01035                 dims[1] = (hsize_t)(area->get_height()?area->get_height():1);
01036                 dims[2] = (hsize_t)(area->get_depth()?area->get_depth():1);
01037 
01038                 hid_t memoryspace = H5Screate_simple(3, dims, NULL);
01039                 switch(dt) {
01040                 case EMUtil::EM_FLOAT:
01041                         H5Dwrite(ds, H5T_NATIVE_FLOAT, memoryspace, spc, H5P_DEFAULT, data);
01042                         break;
01043                 default:
01044                         throw ImageWriteException(filename,"HDF5 does not support regional writing for this data format");
01045                 }
01046                 H5Sclose(memoryspace);
01047         }
01048         else {
01049                 switch(dt) {
01050                 case EMUtil::EM_FLOAT:
01051                         H5Dwrite(ds,H5T_NATIVE_FLOAT,spc,spc,H5P_DEFAULT,data);
01052                         break;
01053                 case EMUtil::EM_USHORT:
01054                         usdata = new unsigned short[size];
01055                         for (size_t i = 0; i < size; ++i) {
01056                                 if(data[i] <= rendermin) {
01057                                         usdata[i] = 0;
01058                                 }
01059                                 else if(data[i] >= rendermax) {
01060                                         usdata[i] = USHRT_MAX;
01061                                 }
01062                                 else {
01063                                         usdata[i]=(unsigned short)((data[i]-rendermin)/(rendermax-rendermin)*USHRT_MAX);
01064                                 }
01065                         }
01066                         H5Dwrite(ds,H5T_NATIVE_USHORT,spc,spc,H5P_DEFAULT,usdata);
01067                         if(usdata) {delete [] usdata; usdata=0;}
01068                         break;
01069                 case EMUtil::EM_UCHAR:
01070                         cdata = new unsigned char[size];
01071                         for (size_t i = 0; i < size; ++i) {
01072                                 if(data[i] <= rendermin) {
01073                                         cdata[i] = 0;
01074                                 }
01075                                 else if(data[i] >= rendermax){
01076                                         cdata[i] = UCHAR_MAX;
01077                                 }
01078                                 else {
01079                                         cdata[i]=(unsigned char)((data[i]-rendermin)/(rendermax-rendermin)*UCHAR_MAX);
01080                                 }
01081                         }
01082                         H5Dwrite(ds,H5T_NATIVE_UCHAR,spc,spc,H5P_DEFAULT,cdata);
01083                         if(cdata) {delete [] cdata; cdata=0;}
01084                         break;
01085                 default:
01086                         throw ImageWriteException(filename,"HDF5 does not support this data format");
01087                 }
01088         }
01089 
01090         H5Sclose(spc);
01091         H5Dclose(ds);
01092         EXITFUNC;
01093         return 0;
01094 }
01095 
01096 int HdfIO2::get_nimg()
01097 {
01098         init();
01099         hid_t attr=H5Aopen_name(group,"imageid_max");
01100         int n = read_attr(attr);
01101         H5Aclose(attr);
01102 
01103         return n+1;
01104 }
01105 
01106 void HdfIO2::flush()
01107 {
01108         return;
01109 }
01110 
01111 bool HdfIO2::is_complex_mode()
01112 {
01113         return false;
01114 }
01115 
01116 // always big endian
01117 bool HdfIO2::is_image_big_endian()
01118 {
01119         return true;
01120 }
01121 
01122 
01123 
01124 #endif  //EM_HDF5

Generated on Thu Dec 9 13:45:46 2010 for EMAN2 by  doxygen 1.3.9.1