Main Page | Modules | Namespace List | Class Hierarchy | Alphabetical List | Class List | Directories | File List | Namespace Members | Class Members | File Members

hdfio2.cpp

Go to the documentation of this file.
00001 
00005 /*
00006  * Author: Steven Ludtke, 04/10/2003 (sludtke@bcm.edu)
00007  * Copyright (c) 2000-2006 Baylor College of Medicine
00008  *
00009  * This software is issued under a joint BSD/GNU license. You may use the
00010  * source code in this file under either license. However, note that the
00011  * complete EMAN2 and SPARX software packages have some GPL dependencies,
00012  * so you are responsible for compliance with the licenses of these packages
00013  * if you opt to use BSD licensing. The warranty disclaimer below holds
00014  * in either instance.
00015  *
00016  * This complete copyright notice must be included in any revised version of the
00017  * source code. Additional authorship citations may be added, but existing
00018  * author citations must be preserved.
00019  *
00020  * This program is free software; you can redistribute it and/or modify
00021  * it under the terms of the GNU General Public License as published by
00022  * the Free Software Foundation; either version 2 of the License, or
00023  * (at your option) any later version.
00024  *
00025  * This program is distributed in the hope that it will be useful,
00026  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00027  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
00028  * GNU General Public License for more details.
00029  *
00030  * You should have received a copy of the GNU General Public License
00031  * along with this program; if not, write to the Free Software
00032  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
00033  *
00034  * */
00035 
00036 #ifdef EM_HDF5
00037 
00038 //#define DEBUGHDF      1
00039 
00040 #include "hdfio2.h"
00041 #include "geometry.h"
00042 #include "ctf.h"
00043 #include "emassert.h"
00044 #include "transform.h"
00045 #include "ctf.h"
00046 #include <iostream>
00047 #include <cstring>
00048 
00049 #ifndef WIN32
00050         #include <sys/param.h>
00051 #else
00052         #define  MAXPATHLEN (MAX_PATH * 4)
00053 #endif  //WIN32
00054 
00055 using namespace EMAN;
00056 
00057 static const int ATTR_NAME_LEN = 128;
00058 
00059 HdfIO2::HdfIO2(const string & hdf_filename, IOMode rw)
00060 :       nx(1), ny(1), nz(1), is_exist(false),
00061         file(-1), group(-1), filename(hdf_filename),
00062         rw_mode(rw), initialized(false)
00063 {
00064         accprop=H5Pcreate(H5P_FILE_ACCESS);
00065 
00066         //STDIO file driver has 2G size limit on 32 bit Linux system
00067         H5Pset_fapl_sec2( accprop );
00068         //H5Pset_fapl_stdio( accprop );
00069 
00070 //      H5Pset_fapl_core( accprop, 1048576, 0  );
00071 //      H5Pset_cache(accprop)
00072         hsize_t dims=1;
00073         simple_space=H5Screate_simple(1,&dims,NULL);
00074 
00075         meta_attr_dict = Dict();
00076 }
00077 
00078 HdfIO2::~HdfIO2()
00079 {
00080         H5Sclose(simple_space);
00081         H5Pclose(accprop);
00082     if (group >= 0) {
00083         H5Gclose(group);
00084     }
00085     if (file >= 0) {
00086                 H5Fflush(file,H5F_SCOPE_GLOBAL);        // If there were no resource leaks, this wouldn't be necessary...
00087                 H5Fclose(file);
00088     }
00089 #ifdef DEBUGHDF
00090         printf("HDf close\n");
00091 #endif
00092 }
00093 
00094 // This reads an already opened attribute and returns the results as an EMObject
00095 // The attribute is not closed
00096 EMObject HdfIO2::read_attr(hid_t attr) {
00097         hid_t type = H5Aget_type(attr);
00098         hid_t spc = H5Aget_space(attr);
00099         H5T_class_t cls = H5Tget_class(type);
00100         size_t sz = H5Tget_size(type);                                          // storage size, arrays handled in the 'space'
00101         hssize_t pts = H5Sget_simple_extent_npoints(spc);       // number of points > 1 if an array of floats or integers
00102 
00103         EMObject ret(0);
00104         char c;
00105         int i;
00106 //      unsigned int ui;
00107         float f,*fa;
00108         int * ia;
00109 //      unsigned int * uia;
00110         double d;
00111         char *s;
00112         vector <float> fv((size_t)pts);
00113         vector <int> iv((size_t)pts);
00114 //      vector <unsigned int> uiv(pts);
00115 
00116         float *matrix;
00117         Transform* t;
00118         Ctf* ctf;
00119 //      int r, c, k=0;
00120 
00121         switch (cls) {
00122         case H5T_INTEGER:
00123                 if(sz==1) {
00124                         H5Aread(attr,H5T_NATIVE_CHAR,&c);
00125                         bool b = false;
00126                         if(c=='T') {
00127                                 b = true;
00128                         }
00129                         else if(c=='F') {
00130                                 b = false;
00131                         }
00132                         ret = EMObject(b);
00133                 }
00134                 else if(sz==4) {
00135                         if(pts==1) {
00136                                 H5Aread(attr,H5T_NATIVE_INT,&i);
00137                                 ret=EMObject(i);
00138                         }
00139                         else {
00140                                 ia=(int *)malloc((size_t)pts*sizeof(int));
00141                                 H5Aread(attr,H5T_NATIVE_INT,ia);
00142                                 for (i=0; i<pts; i++) iv[i]=ia[i];
00143                                 free(ia);
00144                                 ret=EMObject(iv);
00145                         }
00146                 }
00147                 break;
00148 //      case H5T_UNSIGNED_INTEGER:
00149 //              if(pts==1) {
00150 //                      H5Aread(attr,H5T_NATIVE_UINT,&ui);
00151 //                      ret=EMObject(ui);
00152 //              }
00153 //              else {
00154 //                      uia=(unsigned int *)malloc(pts*sizeof(unsigned int));
00155 //                      H5Aread(attr,H5T_NATIVE_UINT,uia);
00156 //                      for (i=0; i<pts; i++) uiv[i]=uia[i];
00157 //                      free(uia);
00158 //                      ret=EMObject(uiv);
00159 //              }
00160 //              break;
00161         case H5T_FLOAT:
00162                 if (sz==4) {
00163                         if (pts==1) {
00164                                 H5Aread(attr,H5T_NATIVE_FLOAT,&f);
00165                                 ret=EMObject(f);
00166                         }
00167                         else {
00168                                 fa=(float *)malloc((size_t)pts*sizeof(float));
00169                                 H5Aread(attr,H5T_NATIVE_FLOAT,fa);
00170                                 for (i=0; i<pts; i++) fv[i]=fa[i];
00171                                 free(fa);
00172                                 ret=EMObject(fv);
00173                         }
00174                 }
00175                 else if (sz==8) {
00176                         H5Aread(attr,H5T_NATIVE_DOUBLE,&d);
00177                         ret=EMObject(d);
00178                 }
00179                 break;
00180         case H5T_STRING:
00181                 s=(char *)malloc(sz+1);
00182                 H5Aread(attr,type,s);
00183 //              H5Aread(attr,H5T_NATIVE_CHAR,s);
00184                 if(s[0] == 'O' && isdigit(s[1])) {
00185                         ctf = new EMAN1Ctf();
00186                         ctf->from_string(string(s));
00187                         ret = EMObject(ctf);
00188                         delete ctf;
00189                 }
00190                 else if(s[0] == 'E' && isdigit(s[1])) {
00191                         ctf = new EMAN2Ctf();
00192                         ctf->from_string(string(s));
00193                         ret = EMObject(ctf);
00194                         delete ctf;
00195                 }
00196                 else {
00197                         ret=EMObject(s);
00198                 }
00199                 free(s);
00200                 break;
00201         case H5T_COMPOUND:
00202                 matrix = (float*)malloc(12*sizeof(float));
00203                 H5Aread(attr, type, matrix);
00204 //              ret.create_transform3d_by_array(trans3d);
00205                 t = new Transform(matrix);
00206                 ret = EMObject(t);
00207                 free(matrix);
00208                 delete t; t=0;
00209 
00210 //              trans3d = (float*)malloc(16*sizeof(float));     //16 float for a Transform3D object
00211 //              H5Aread(attr, type, trans3d);
00212 // //           ret.create_transform3d_by_array(trans3d);
00213 //              trans = new Transform3D();
00214 //              for(r=0; r<4; ++r) {
00215 //                      for(c=0; c<4; ++c) {
00216 //                              trans->set(r, c, trans3d[k]);
00217 //                              ++k;
00218 //                      }
00219 //              }
00220 //              ret = EMObject(trans);
00221 //              free(trans3d);
00222                 break;
00223         default:
00224                 LOGERR("Unhandled HDF5 metadata %d", cls);
00225         }
00226 
00227         H5Sclose(spc);
00228         H5Tclose(type);
00229 
00230         return ret;
00231 }
00232 
00233 // This writes an attribute with specified name to a given open object
00234 // The attribute is opened and closed. returns 0 on success
00235 int HdfIO2::write_attr(hid_t loc,const char *name,EMObject obj) {
00236         hid_t type=0;
00237         hid_t spc=0;
00238         hsize_t dims=1;
00239         vector <float> fv;
00240         vector <int> iv;
00241         switch(obj.get_type())
00242         {
00243         case EMObject::BOOL:
00244                 type=H5Tcopy(H5T_NATIVE_CHAR);
00245                 spc=H5Scopy(simple_space);
00246                 break;
00247         case EMObject::INT:
00248                 type=H5Tcopy(H5T_NATIVE_INT);
00249                 spc=H5Scopy(simple_space);
00250                 break;
00251         case EMObject::UNSIGNEDINT:
00252                 type=H5Tcopy(H5T_NATIVE_UINT);
00253                 spc=H5Scopy(simple_space);
00254                 break;
00255         case EMObject::FLOAT:
00256                 type=H5Tcopy(H5T_NATIVE_FLOAT);
00257                 spc=H5Scopy(simple_space);
00258                 break;
00259         case EMObject::DOUBLE:
00260                 type=H5Tcopy(H5T_NATIVE_DOUBLE);
00261                 spc=H5Scopy(simple_space);
00262                 break;
00263         case EMObject::STRING:
00264         case EMObject::CTF:
00265                 type=H5Tcopy(H5T_C_S1);
00266                 H5Tset_size(type,strlen((const char *)obj)+1);
00267                 spc=H5Screate(H5S_SCALAR);
00268                 break;
00269         case EMObject::FLOATARRAY:
00270                 type=H5Tcopy(H5T_NATIVE_FLOAT);
00271                 fv=obj;
00272                 dims=fv.size();
00273                 spc=H5Screate_simple(1,&dims,NULL);
00274                 break;
00275         case EMObject::INTARRAY:
00276                 type=H5Tcopy(H5T_NATIVE_INT);
00277                 iv=obj;
00278                 dims=iv.size();
00279                 spc=H5Screate_simple(1,&dims,NULL);
00280                 break;
00281         case EMObject::TRANSFORM:
00282                 type = H5Tcreate(H5T_COMPOUND, 12 * sizeof(float)); //Transform is a 3x4 matrix
00283                 H5Tinsert(type, "00", 0, H5T_NATIVE_FLOAT);
00284                 H5Tinsert(type, "01", 1*sizeof(float), H5T_NATIVE_FLOAT);
00285                 H5Tinsert(type, "02", 2*sizeof(float), H5T_NATIVE_FLOAT);
00286                 H5Tinsert(type, "03", 3*sizeof(float), H5T_NATIVE_FLOAT);
00287                 H5Tinsert(type, "10", 4*sizeof(float), H5T_NATIVE_FLOAT);
00288                 H5Tinsert(type, "11", 5*sizeof(float), H5T_NATIVE_FLOAT);
00289                 H5Tinsert(type, "12", 6*sizeof(float), H5T_NATIVE_FLOAT);
00290                 H5Tinsert(type, "13", 7*sizeof(float), H5T_NATIVE_FLOAT);
00291                 H5Tinsert(type, "20", 8*sizeof(float), H5T_NATIVE_FLOAT);
00292                 H5Tinsert(type, "21", 9*sizeof(float), H5T_NATIVE_FLOAT);
00293                 H5Tinsert(type, "22", 10*sizeof(float), H5T_NATIVE_FLOAT);
00294                 H5Tinsert(type, "23", 11*sizeof(float), H5T_NATIVE_FLOAT);
00295                 H5Tpack(type);
00296 
00297                 dims = 1;       //one compound type
00298                 spc = H5Screate_simple(1, &dims, NULL);
00299                 break;
00300         case EMObject::TRANSFORMARRAY:
00301         case EMObject::STRINGARRAY:
00302         case EMObject::EMDATA:
00303         case EMObject::XYDATA:
00304         case EMObject::FLOAT_POINTER:
00305         case EMObject::INT_POINTER:
00306         case EMObject::VOID_POINTER:
00307                 return -1;
00308                 break;
00309         case EMObject::UNKNOWN:
00310                 break;
00311         }
00312 
00313     //we need this delete attribute call here, even we called erase_header()
00314     //at the biginning of write_header(), since the  "imageid_max" need be updated correctly.
00315         if( H5Adelete(loc,name) < 0 ) {
00316 #ifdef DEBUGHDF
00317                 LOGERR("Attribute %s deletion error in write_attr().\n", name);
00318 #endif
00319         }
00320         else {
00321 #ifdef DEBUGHDF
00322                 printf("delete attribute %s successfully in write_attr().\n", name);
00323 #endif
00324         }
00325         hid_t attr = H5Acreate(loc,name,type,spc,H5P_DEFAULT);
00326 
00327         bool b;
00328         char c;
00329         int i;
00330         float f,*fa;
00331         int * ia;
00332         unsigned int ui;
00333         double d;
00334         const char *s;
00335         Transform * tp;
00336         switch(obj.get_type()) {
00337         case EMObject::BOOL:
00338                 b = (bool)obj;
00339                 if(b) {
00340                         c = 'T';
00341                 } else {
00342                         c = 'F';
00343                 }
00344                 H5Awrite(attr,type,&c);
00345                 break;
00346         case EMObject::INT:
00347                 i=(int)obj;
00348                 H5Awrite(attr,type,&i);
00349                 break;
00350         case EMObject::UNSIGNEDINT:
00351                 ui=(unsigned int)obj;
00352                 H5Awrite(attr,type,&ui);
00353                 break;
00354         case EMObject::FLOAT:
00355                 f=(float)obj;
00356                 H5Awrite(attr,type,&f);
00357                 break;
00358         case EMObject::DOUBLE:
00359                 d=(double)obj;
00360                 H5Awrite(attr,type,&d);
00361                 break;
00362         case EMObject::STRING:
00363         case EMObject::CTF:
00364                 s=(const char *)obj;
00365                 H5Awrite(attr,type,s);
00366                 break;
00367         case EMObject::FLOATARRAY:
00368                 fa=(float *)malloc(fv.size()*sizeof(float));
00369                 for (ui=0; ui<fv.size(); ui++) fa[ui]=fv[ui];
00370                 H5Awrite(attr,type,fa);
00371                 free(fa);
00372                 break;
00373         case EMObject::INTARRAY:
00374                 ia=(int *)malloc(iv.size()*sizeof(int));
00375                 for (ui=0; ui<iv.size(); ui++) ia[ui]=iv[ui];
00376                 H5Awrite(attr,type,ia);
00377                 free(ia);
00378                 break;
00379         case EMObject::TRANSFORM:
00380         {
00381                 tp = (Transform *)obj;
00382                 fa = (float *)malloc(12*sizeof(float));
00383                 int r, c, k=0;
00384                 for(r=0; r<3; ++r) {
00385                         for(c=0; c<4; ++c) {
00386                                 fa[k] = tp->at(r,c);
00387                                 ++k;
00388                         }
00389                 }
00390                 H5Awrite(attr,type,fa);
00391                 free(fa);
00392         }
00393                 break;
00394 //      case EMObject::STRINGARRAY:
00395 //      case EMObject::EMDATA:
00396 //      case EMObject::XYDATA:
00397 //              return -1;
00398 //              break;
00399         default:
00400                 LOGERR("Unhandled HDF5 metadata '%s'", name);
00401 
00402         }
00403 
00404         herr_t ret1 = H5Tclose(type);
00405         herr_t ret2 = H5Sclose(spc);
00406         herr_t ret3 = H5Aclose(attr);
00407         if(ret1>=0 && ret2>=0 && ret3>=0) {
00408                 return 0;
00409         }
00410         else {
00411                 LOGERR("close error in write_attr()\n");
00412                 return -1;
00413         }
00414 }
00415 
00416 // Initializes the file for read-only or read-write access
00417 // Data is stored under /MDF/images
00418 // An attribute named imageid_max stores the number of the highest
00419 // numbered image in the file.
00420 // A group is then made for each individual image, all metadata for the
00421 // individual images is currently associated with the GROUP, not the dataset
00422 // dataset-specific data could also be associated with the dataset in
00423 // future. At the moment, there is only a single dataset in each group.
00424 void HdfIO2::init()
00425 {
00426         ENTERFUNC;
00427         if (initialized) {
00428                 return;
00429         }
00430 #ifdef DEBUGHDF
00431         printf("init\n");
00432 #endif
00433 
00434         H5Eset_auto(0, 0);      // Turn off console error logging.
00435 
00436         if (rw_mode == READ_ONLY) {
00437                 file = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, accprop);
00438                 if (file<0) throw FileAccessException(filename);
00439         }
00440         else {
00441                 file = H5Fopen(filename.c_str(), H5F_ACC_RDWR, accprop);
00442                 if (file < 0) {
00443                         file = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, accprop);
00444                         if (file < 0) {
00445                                 throw FileAccessException(filename);
00446                         }
00447                         else {
00448 #ifdef DEBUGHDF
00449                                 printf("File truncated or new file created\n");
00450 #endif
00451                         }
00452                 }
00453         }
00454 
00455         group=H5Gopen(file,"/MDF/images");
00456         if (group<0) {
00457                 if (rw_mode == READ_ONLY) throw ImageReadException(filename,"HDF5 file has no image data (no /MDF group)");
00458                 group=H5Gcreate(file,"/MDF",64);                // create the group for Macromolecular data
00459                 if (group<0) throw ImageWriteException(filename,"Unable to add image group (/MDF) to HDF5 file");
00460                 H5Gclose(group);
00461                 group=H5Gcreate(file,"/MDF/images",4096);               // create the group for images/volumes
00462                 if (group<0) throw ImageWriteException(filename,"Unable to add image group (/MDF/images) to HDF5 file");
00463                 write_attr(group,"imageid_max",EMObject(-1));
00464         }
00465         else {  //read the meta attributes for all images
00466                 int nattr=H5Aget_num_attrs(group);
00467 
00468                 char name[ATTR_NAME_LEN];
00469                 for (int i=0; i<nattr; i++) {
00470                         hid_t attr=H5Aopen_idx(group, i);
00471                         H5Aget_name(attr,127,name);
00472 
00473                         EMObject val=read_attr(attr);
00474                         meta_attr_dict["DDD."+string(name)]=val;
00475 
00476                         H5Aclose(attr);
00477                 }
00478 
00479         }
00480         initialized = true;
00481         EXITFUNC;
00482 }
00483 
00484 
00485 // If this version of init() returns -1, then we have an old-style HDF5 file
00486 int HdfIO2::init_test()
00487 {
00488         ENTERFUNC;
00489         if (initialized) {
00490                 return 1;
00491         }
00492 #ifdef DEBUGHDF
00493         printf("init_test\n");
00494 #endif
00495 
00496         H5Eset_auto(0, 0);      // Turn off console error logging.
00497 
00498         hid_t fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5Pcreate(H5P_FILE_ACCESS));
00499         hid_t groupid = H5Gopen(fileid, "/");
00500         hid_t attid = H5Aopen_name(groupid, "num_dataset");
00501 
00502         if (attid < 0) {
00503                 H5Gclose(groupid);
00504                 H5Fclose(fileid);
00505                 init();
00506                 EXITFUNC;
00507                 return 0;
00508         }
00509         else {
00510                 H5Aclose(attid);
00511                 H5Gclose(groupid);
00512                 H5Fclose(fileid);
00513                 EXITFUNC;
00514                 return -1;
00515         }
00516 }
00517 
00518 bool HdfIO2::is_valid(const void *first_block)
00519 {
00520         ENTERFUNC;
00521 
00522         if (first_block) {
00523                 char signature[8] = { 137,72,68,70,13,10,26,10 };
00524                 if (strncmp((const char *)first_block,signature,8)==0) return true;
00525                 // const char* f=(const char *)first_block;
00526                 // printf("bad hdf signature %d %d %d %d %d %d %d %d",f[0],f[1],f[2],f[3],f[4],f[5],f[6],f[7]);
00527                 return false;
00528         }
00529         EXITFUNC;
00530         return false;
00531 }
00532 
00533 // Reads all of the attributes from the /MDF/images/<imgno> group
00534 int HdfIO2::read_header(Dict & dict, int image_index, const Region * area, bool)
00535 {
00536         ENTERFUNC;
00537         init();
00538 
00540         size_t meta_attr_size = meta_attr_dict.size();
00541         if(meta_attr_size!=0) {
00542                 for (size_t i=0; i<meta_attr_size; ++i) {
00543                         dict[meta_attr_dict.keys()[i]] = meta_attr_dict.values()[i];
00544                 }
00545         }
00546 
00547 #ifdef DEBUGHDF
00548         printf("read_head %d\n", image_index);
00549 #endif
00550         int i;
00551         // Each image is in a group for later expansion. Open the group
00552         char ipath[50];
00553         sprintf(ipath,"/MDF/images/%d", image_index);
00554         hid_t igrp=H5Gopen(file, ipath);
00555 
00556         int nattr=H5Aget_num_attrs(igrp);
00557 
00558         char name[ATTR_NAME_LEN];
00559         for (i=0; i<nattr; i++) {
00560                 hid_t attr=H5Aopen_idx(igrp, i);
00561                 H5Aget_name(attr,127,name);
00562                 if (strncmp(name,"EMAN.", 5)!=0) {
00563                         H5Aclose(attr);
00564                         continue;
00565                 }
00566                 EMObject val=read_attr(attr);
00567                 dict[name+5]=val;
00568                 H5Aclose(attr);
00569         }
00570 
00571         if(dict.has_key("ctf")) {
00572                 string ctfString = (string)dict["ctf"];
00573                 if(ctfString.substr(0, 1) == "O") {
00574                         Ctf * ctf_ = new EMAN1Ctf();
00575                         ctf_->from_string(ctfString);
00576                         dict.erase("ctf");
00577                         dict["ctf"] = ctf_;
00578                         delete ctf_;
00579                 }
00580                 else if(ctfString.substr(0, 1) == "E") {
00581                         Ctf * ctf_ = new EMAN2Ctf();
00582                         ctf_->from_string(ctfString);
00583                         dict.erase("ctf");
00584                         dict["ctf"] = ctf_;
00585                         delete ctf_;
00586                 }
00587         }
00588 
00589         if(area) {
00590                 check_region(area, IntSize(dict["nx"], dict["ny"], dict["nz"]), false, false);
00591 
00592                 dict["nx"] = area->get_width();
00593                 dict["ny"] = area->get_height();
00594                 dict["nz"] = area->get_depth();
00595 
00596                 if( dict.has_key("apix_x") && dict.has_key("apix_y") && dict.has_key("apix_z") )
00597                 {
00598                         if( dict.has_key("origin_x") && dict.has_key("origin_y") && dict.has_key("origin_z") )
00599                         {
00600                                 float xorigin = dict["origin_x"];
00601                                 float yorigin = dict["origin_y"];
00602                                 float zorigin = dict["origin_z"];
00603 
00604                                 float apix_x = dict["apix_x"];
00605                                 float apix_y = dict["apix_y"];
00606                                 float apix_z = dict["apix_z"];
00607 
00608                                 dict["origin_x"] = xorigin + apix_x * area->origin[0];
00609                                 dict["origin_y"] = yorigin + apix_y * area->origin[1];
00610                                 dict["origin_z"] = zorigin + apix_z * area->origin[2];
00611                         }
00612                 }
00613         }
00614 
00615         H5Gclose(igrp);
00616 
00617         //Get the data type from data set, HDF5 file header attribute 'datatype' may be wrong
00618         sprintf(ipath,"/MDF/images/%d/image",image_index);
00619         hid_t ds=H5Dopen(file,ipath);
00620 
00621         if(ds>0) {      //ds>0 means successfully open the dataset
00622                 hid_t dt = H5Dget_type(ds);
00623 
00624                 switch(H5Tget_size(dt)) {
00625                 case 4:
00626                         dict["datatype"] = (int)EMUtil::EM_FLOAT;
00627                         break;
00628                 case 2:
00629                         dict["datatype"] = (int)EMUtil::EM_USHORT;
00630                         break;
00631                 case 1:
00632                         dict["datatype"] = (int)EMUtil::EM_UCHAR;
00633                         break;
00634                 default:
00635                         throw ImageReadException(filename, "EMAN does not support this data type.");
00636                 }
00637 
00638                 H5Tclose(dt);
00639         }
00640 
00641         H5Dclose(ds);
00642 
00643         EXITFUNC;
00644         return 0;
00645 }
00646 
00647 // This erases any existing attributes from the image group
00648 // prior to writing a new header. For a new image there
00649 // won't be any, so this should be harmless.
00650 int HdfIO2::erase_header(int image_index)
00651 {
00652         ENTERFUNC;
00653 
00654         if(image_index < 0) return 0; //image_index<0 for appending image, no need for erasing
00655 
00656         init();
00657 #ifdef DEBUGHDF
00658         printf("erase_head %d\n",image_index);
00659 #endif
00660         int i;
00661         // Each image is in a group for later expansion. Open the group
00662         char ipath[50];
00663         sprintf(ipath,"/MDF/images/%d", image_index);
00664         hid_t igrp=H5Gopen(file, ipath);
00665 
00666         int nattr=H5Aget_num_attrs(igrp);
00667 
00668         char name[ATTR_NAME_LEN];
00669         for (i=0; i<nattr; i++) {
00670                 hid_t attr = H5Aopen_idx(igrp, 0); //use 0 as index here, since the H5Adelete() will change the index
00671                 H5Aget_name(attr,127,name);
00672                 H5Aclose(attr);
00673                 if( H5Adelete(igrp,name) < 0 ) {
00674                         LOGERR("attribute %s deletion error in erase_header().\n", name);
00675                 }
00676         }
00677 
00678         H5Gclose(igrp);
00679         EXITFUNC;
00680         return 0;
00681 }
00682 
00683 
00684 int HdfIO2::read_data(float *data, int image_index, const Region *area, bool)
00685 {
00686         ENTERFUNC;
00687 #ifdef DEBUGHDF
00688         printf("read_data %d\n",image_index);
00689 #endif
00690 
00691         char ipath[50];
00692         sprintf(ipath,"/MDF/images/%d/image",image_index);
00693         hid_t ds=H5Dopen(file,ipath);
00694         if (ds<0) throw ImageWriteException(filename,"Image does not exist");
00695         hid_t spc=H5Dget_space(ds);
00696         hid_t dt = H5Dget_type(ds);
00697 
00698         hsize_t dims_out[3];
00699         hsize_t rank = H5Sget_simple_extent_ndims(spc);
00700 
00701         H5Sget_simple_extent_dims(spc, dims_out, NULL);
00702         if(rank == 1) {
00703                 nx = dims_out[0];
00704                 ny = 1;
00705                 nz = 1;
00706         }
00707         else if(rank == 2) {
00708                 nx = dims_out[1];
00709                 ny = dims_out[0];
00710                 nz = 1;
00711         }
00712         else if(rank == 3) {
00713                 nx = dims_out[2];
00714                 ny = dims_out[1];
00715                 nz = dims_out[0];
00716         }
00717 
00718         if (area) {
00719                 hid_t memoryspace = 0;
00720 
00721                 /*Get the file dataspace - the region we want to read in the file*/
00722                 int x0 = 0, y0 = 0, z0 = 0;             //the coordinates for up left corner, trim to be within image bound
00723                 int x1 = 0, y1 = 0, z1 = 0;             //the coordinates for down right corner, trim to be within image bound
00724                 int nx1 = 1, ny1 = 1, nz1 = 1;  //dimensions of the sub-region, actual region read form file
00725                 if(rank == 3) {
00726                         hsize_t     doffset[3];             /* hyperslab offset in the file */
00727                         doffset[2] = (hsize_t)(area->x_origin() < 0 ? 0 : area->x_origin());
00728                         doffset[1] = (hsize_t)(area->y_origin() < 0 ? 0 : area->y_origin());
00729                         doffset[0] = (hsize_t)(area->z_origin() < 0 ? 0 : area->z_origin());
00730                         x0 = (int)doffset[0];
00731                         y0 = (int)doffset[1];
00732                         z0 = (int)doffset[2];
00733 
00734                         z1 = (int)(area->x_origin() + area->get_width());
00735                         z1 = (int)(z1 > static_cast<int>(nx) ? nx : z1);
00736 
00737                         y1 = (int)(area->y_origin() + area->get_height());
00738                         y1 = (int)(y1 > static_cast<int>(ny) ? ny : y1);
00739                         if(y1 <= 0) {
00740                                 y1 = 1;
00741                         }
00742 
00743                         x1 = (int)(area->z_origin() + area->get_depth());
00744                         x1 = (int)(x1 > static_cast<int>(nz) ? nz : x1);
00745                         if(x1 <= 0) {
00746                                 x1 = 1;
00747                         }
00748 
00749                         if(x1 < x0 || y1< y0 || z1 < z0) return 0; //out of bounds, this is fine, nothing happens
00750 
00751                         hsize_t     dcount[3];              /* size of the hyperslab in the file */
00752                         dcount[0] = x1 - doffset[0];
00753                         dcount[1] = y1 - doffset[1];
00754                         dcount[2] = z1 - doffset[2];
00755 
00756                         H5Sselect_hyperslab (spc, H5S_SELECT_SET, (const hsize_t*)doffset, NULL, (const hsize_t*)dcount, NULL);
00757 
00758                         /*Define memory dataspace - the memory we will created for the region*/
00759                         hsize_t     dims[3];              /* size of the region in the memory */
00760                         dims[0] = dcount[2]?dcount[2]:1;
00761                         dims[1] = dcount[1]?dcount[1]:1;
00762                         dims[2] = dcount[0]?dcount[0]:1;
00763                         nx1 = (int)dims[0];
00764                         ny1 = (int)dims[1];
00765                         nz1 = (int)dims[2];
00766 
00767                         memoryspace = H5Screate_simple(3, dims, NULL);
00768                 }
00769                 else if(rank == 2) {
00770                         hsize_t     doffset[2];             /* hyperslab offset in the file */
00771                         doffset[1] = (hsize_t)(area->x_origin() < 0 ? 0 : area->x_origin());
00772                         doffset[0] = (hsize_t)(area->y_origin() < 0 ? 0 : area->y_origin());
00773                         x0 = (int)doffset[0];
00774                         y0 = (int)doffset[1];
00775                         z0 = 1;
00776 
00777                         y1 = (int)(area->x_origin() + area->get_width());
00778                         y1 = (int)(y1 > static_cast<int>(nx) ? nx : y1);
00779 
00780                         x1 = (int)(area->y_origin() + area->get_height());
00781                         x1 = (int)(x1 > static_cast<int>(ny) ? ny : x1);
00782                         if(x1 <= 0) {
00783                                 x1 = 1;
00784                         }
00785 
00786                         z1 = 1;
00787 
00788                         if(x1 < x0 || y1< y0) return 0; //out of bounds, this is fine, nothing happens
00789 
00790                         hsize_t     dcount[2];              /* size of the hyperslab in the file */
00791                         dcount[0] = x1 - doffset[0];
00792                         dcount[1] = y1 - doffset[1];
00793 
00794                         H5Sselect_none(spc);
00795                         H5Sselect_hyperslab (spc, H5S_SELECT_SET, (const hsize_t*)doffset, NULL, (const hsize_t*)dcount, NULL);
00796 
00797                         /*Define memory dataspace - the memory we will created for the region*/
00798                         hsize_t     dims[2];              /* size of the region in the memory */
00799                         dims[0] = (hsize_t)(dcount[1]?dcount[1]:1);
00800                         dims[1] = (hsize_t)(dcount[0]?dcount[0]:1);
00801                         nx1 = (int)dims[0];
00802                         ny1 = (int)dims[1];
00803                         nz1 = 1;
00804 
00805                         memoryspace = H5Screate_simple(2, dims, NULL);
00806                 }
00807 
00808                 if( (area->x_origin()>=0) && (area->y_origin()>=0) && (area->z_origin()>=0) && ((hsize_t)(area->x_origin() + area->get_width())<=nx) && ((hsize_t)(area->y_origin() + area->get_height())<=ny) && ((hsize_t)(area->z_origin() + area->get_depth())<=nz) ){      //the region is in boundary
00809                         H5Dread(ds,H5T_NATIVE_FLOAT,memoryspace,spc,H5P_DEFAULT,data);
00810                 }
00811                 else {  //the region are partial out of boundary
00812                         /* When the requested region has some part out of image boundary,
00813                          * we need read the sub-area which is within image,
00814                          * and fill the out of boundary part with zero.
00815                          * We actually read the sub-region from HDF by hyperslab I/O,
00816                          * then copy it back to the pre-allocated region.*/
00817                         float * subdata = new float[nx1*ny1*nz1];
00818 
00819 
00820                         H5Dread(ds,H5T_NATIVE_FLOAT,memoryspace,spc,H5P_DEFAULT,subdata);
00821 
00822                         int xd0=0, yd0=0, zd0=0;        //The coordinates of the top-left corner sub-region in region
00823                         size_t clipped_row_size = 0;
00824                         if(rank == 3) {
00825                                 xd0 = (int) (area->x_origin() < 0 ? -area->x_origin() : 0);
00826                                 yd0 = (int) (area->y_origin() < 0 ? -area->y_origin() : 0);
00827                                 zd0 = (int) (area->z_origin() < 0 ? -area->z_origin() : 0);
00828                                 clipped_row_size = (z1-z0)* sizeof(float);
00829                         }
00830                         else if(rank == 2) {
00831                                 xd0 = (int) (area->x_origin() < 0 ? -area->x_origin() : 0);
00832                                 yd0 = (int) (area->y_origin() < 0 ? -area->y_origin() : 0);
00833                                 clipped_row_size = (y1-y0)* sizeof(float);
00834                         }
00835 
00836                         int src_secsize = nx1 * ny1;
00837                         int dst_secsize = (int)(area->get_width())*(int)(area->get_height());
00838 
00839                         float * src = subdata;
00840                         float * dst = data + zd0*dst_secsize + yd0*(int)(area->get_width()) + xd0;
00841 
00842                         int src_gap = src_secsize - (y1-y0) * nx1;
00843                         int dst_gap = dst_secsize - (y1-y0) * (int)(area->get_width());
00844 
00845                         for(int i = 0; i<nz1; ++i) {
00846                                 for(int j = 0; j<ny1; ++j) {
00847                                         EMUtil::em_memcpy(dst, src, clipped_row_size);
00848 
00849                                         src += nx1;
00850                                         dst += (int)(area->get_width());
00851                                 }
00852                                 src += src_gap;
00853                                 dst += dst_gap;
00854                         }
00855 
00856                         delete [] subdata;
00857                 }
00858                 H5Sclose(memoryspace);
00859         } else {
00860                 hsize_t size = (hsize_t)nx*ny*nz;
00861                 hsize_t i=0;
00862                 hsize_t j=0;
00863                 unsigned short *usdata = (unsigned short *) data;
00864                 unsigned char *cdata = (unsigned char *) data;
00865                 switch(H5Tget_size(dt)) {
00866                 case 4:
00867                         H5Dread(ds,H5T_NATIVE_FLOAT,spc,spc,H5P_DEFAULT,data);
00868                         break;
00869                 case 2:
00870                         H5Dread(ds,H5T_NATIVE_USHORT,spc,spc,H5P_DEFAULT,usdata);
00871                         for (i = 0; i < size; ++i) {
00872                                 j = size - 1 - i;
00873                                 data[j] = static_cast < float >(usdata[j]);
00874                         }
00875                         break;
00876                 case 1:
00877                         H5Dread(ds,H5T_NATIVE_UCHAR,spc,spc,H5P_DEFAULT,cdata);
00878                         for (i = 0; i < size; ++i) {
00879                                 j = size - 1 - i;
00880                                 data[j] = static_cast < float >(cdata[j]);
00881                         }
00882                         break;
00883                 default:
00884                         throw ImageReadException(filename, "EMAN does not support this data type.");
00885                 }
00886         }
00887 
00888         H5Tclose(dt);
00889         H5Sclose(spc);
00890         H5Dclose(ds);
00891         EXITFUNC;
00892         return 0;
00893 }
00894 
00895 
00896 // Writes all attributes in 'dict' to the image group
00897 // Creation of the image dataset is also handled here
00898 int HdfIO2::write_header(const Dict & dict, int image_index, const Region* area,
00899                                                 EMUtil::EMDataType, bool)
00900 {
00901 #ifdef DEBUGHDF
00902         printf("write_head %d\n",image_index);
00903 #endif
00904         ENTERFUNC;
00905         init();
00906 
00907         nx = (int)dict["nx"];
00908         ny = (int)dict["ny"];
00909         nz = (int)dict["nz"];
00910 
00911         if(image_index<0) {
00912                 image_index = get_nimg();
00913         }
00914 
00915         // If image_index<0 append, and make sure the max value in the file is correct
00916         // though this is normally handled by EMData.write_image()
00917         hid_t attr=H5Aopen_name(group,"imageid_max");
00918         int nimg = read_attr(attr);
00919         H5Aclose(attr);
00920 
00921         unsigned int i;
00922         if (image_index<0) image_index=nimg+1;
00923         if (image_index>nimg) {
00924                 write_attr(group,(const char *)"imageid_max",EMObject(image_index));
00925         }
00926 
00927         // Each image is in a group for later expansion. Open the group, create if necessary
00928         char ipath[50];
00929         sprintf(ipath,"/MDF/images/%d",image_index);
00930         hid_t igrp=H5Gopen(file,ipath);
00931 
00932         if (igrp<0) {   //group not existed
00933                 is_exist = false;
00934                 // Need to create a new image group
00935                 igrp=H5Gcreate(file,ipath,64);          // The image is a group, with attributes on the group
00936                 if (igrp<0) throw ImageWriteException(filename,"Unable to add /MDF/images/# to HDF5 file");
00937         }
00940         else {
00941                 is_exist = true;
00942                 int nattr=H5Aget_num_attrs(igrp);
00943                 char name[ATTR_NAME_LEN];
00944                 Dict dict2;
00945                 for (int i=0; i<nattr; i++) {
00946                         hid_t attr=H5Aopen_idx(igrp, i);
00947                         H5Aget_name(attr,127,name);
00948                         if (strncmp(name,"EMAN.", 5)!=0) {
00949                                 H5Aclose(attr);
00950                                 continue;
00951                         }
00952                         EMObject val=read_attr(attr);
00953                         dict2[name+5]=val;
00954                         H5Aclose(attr);
00955 
00956                         if(!dict2.has_key("datatype")) {//by default, HDF5 is written as float
00957                                 dict2["datatype"] = (int)EMUtil::EM_FLOAT;
00958                         }
00959                 }
00960 
00961                 if(area) {
00962                         check_region(area, IntSize(dict2["nx"], dict2["ny"], dict2["nz"]), false, true);
00963                 }
00964                 else {
00965                         erase_header(image_index);
00966 
00967                         //change the size or data type of a image,
00968                         //the existing data set is invalid, unlink it
00969                         if( (int)dict["nx"]*(int)dict["ny"]*(int)dict["nz"] !=
00970                                 (int)dict2["nx"]*(int)dict2["ny"]*(int)dict2["nz"] ||
00971                                 dict["datatype"] != dict2["datatype"] ) {
00972                                 sprintf(ipath,"/MDF/images/%d/image",image_index);
00973                                 H5Gunlink(igrp, ipath);
00974                         }
00975                 }
00976         }
00977 
00978         if(!area) {
00979                 // Write the attributes to the group
00980                 vector <string> keys=dict.keys();
00981 
00982                 for (i=0; i<keys.size(); i++) {
00983                         string s("EMAN.");
00984                         s+=keys[i];
00985                         write_attr(igrp,s.c_str(),dict[keys[i]]);
00986                 }
00987         }
00988 
00989         H5Gclose(igrp);
00990         EXITFUNC;
00991         return 0;
00992 }
00993 
00994 // Writes the actual image data to the corresponding dataset (already created)
00995 int HdfIO2::write_data(float *data, int image_index, const Region* area,
00996                                           EMUtil::EMDataType dt, bool)
00997 {
00998         ENTERFUNC;
00999         if(!data) {
01000                 std::cerr << "Warning:blank image written!!! " << std::endl;
01001                 return 0;
01002         }
01003 
01004 #ifdef DEBUGHDF
01005         printf("write_data %d\n",image_index);
01006 #endif
01007 
01008         if (image_index<0) {
01009                 hid_t attr=H5Aopen_name(group,"imageid_max");
01010                 image_index = read_attr(attr);
01011                 H5Aclose(attr);
01012         }
01013 
01014         hid_t spc;      //dataspace
01015         hid_t ds;       //dataset
01016         char ipath[50];
01017         sprintf(ipath,"/MDF/images/%d/image",image_index);
01018 
01019         // Now create the actual image dataset
01020         if (nz==1)  {
01021                 hsize_t dims[2]= { ny,nx };
01022                 spc=H5Screate_simple(2,dims,NULL);
01023         }
01024         else {
01025                 hsize_t dims[3]= { nz, ny, nx };
01026                 spc=H5Screate_simple(3,dims,NULL);
01027         }
01028 
01029         ds=H5Dopen(file,ipath);
01030         if(ds<0) {//new dataset
01031                 switch(dt) {
01032                 case EMUtil::EM_FLOAT:
01033                         ds=H5Dcreate(file,ipath, H5T_NATIVE_FLOAT, spc, H5P_DEFAULT );
01034                         break;
01035                 case EMUtil::EM_USHORT:
01036                         ds=H5Dcreate(file,ipath, H5T_NATIVE_USHORT, spc, H5P_DEFAULT );
01037                         break;
01038                 case EMUtil::EM_UCHAR:
01039                         ds=H5Dcreate(file,ipath, H5T_NATIVE_UCHAR, spc, H5P_DEFAULT );
01040                         break;
01041                 default:
01042                         throw ImageWriteException(filename,"HDF5 does not support this data format");
01043                 }
01044         }
01045 
01046         //convert data to unsigned short, unsigned char...
01047         hsize_t size = (hsize_t)nx*ny*nz;
01048         unsigned char *cdata = 0;
01049         unsigned short *usdata = 0;
01050         float rendermin = 0.0f;
01051         float rendermax = 0.0f;
01052         EMUtil::getRenderMinMax(data, nx, ny, rendermin, rendermax, nz);
01053 
01054         if(area) {
01055                 hsize_t doffset[3];             /*hyperslab offset in the file*/
01056                 doffset[0] = (hsize_t)(area->x_origin());
01057                 doffset[1] = (hsize_t)(area->y_origin());
01058                 doffset[2] = (hsize_t)(area->z_origin());
01059 
01060                 hsize_t dcount[3];              /*size of the hyperslab in the file*/
01061                 dcount[0] = (hsize_t)(area->get_width());
01062                 dcount[1] = (hsize_t)(area->get_height()?area->get_height():1);
01063                 dcount[2] = (hsize_t)(area->get_depth()?area->get_depth():1);
01064 
01065                 H5Sselect_hyperslab(spc, H5S_SELECT_SET, (const hsize_t*)doffset, NULL, (const hsize_t*)dcount, NULL);
01066 
01067                 /*Create memory space with size of the region.*/
01068                 hsize_t dims[3];        /*size of the region in the memory*/
01069                 dims[0] = (hsize_t)(area->get_width());
01070                 dims[1] = (hsize_t)(area->get_height()?area->get_height():1);
01071                 dims[2] = (hsize_t)(area->get_depth()?area->get_depth():1);
01072 
01073                 hid_t memoryspace = H5Screate_simple(3, dims, NULL);
01074                 switch(dt) {
01075                 case EMUtil::EM_FLOAT:
01076                         H5Dwrite(ds, H5T_NATIVE_FLOAT, memoryspace, spc, H5P_DEFAULT, data);
01077                         break;
01078                 default:
01079                         throw ImageWriteException(filename,"HDF5 does not support regional writing for this data format");
01080                 }
01081                 H5Sclose(memoryspace);
01082         }
01083         else {
01084                 switch(dt) {
01085                 case EMUtil::EM_FLOAT:
01086                         H5Dwrite(ds,H5T_NATIVE_FLOAT,spc,spc,H5P_DEFAULT,data);
01087                         break;
01088                 case EMUtil::EM_USHORT:
01089                         usdata = new unsigned short[size];
01090                         for (size_t i = 0; i < size; ++i) {
01091                                 if(data[i] <= rendermin) {
01092                                         usdata[i] = 0;
01093                                 }
01094                                 else if(data[i] >= rendermax) {
01095                                         usdata[i] = USHRT_MAX;
01096                                 }
01097                                 else {
01098                                         usdata[i]=(unsigned short)((data[i]-rendermin)/(rendermax-rendermin)*USHRT_MAX);
01099                                 }
01100                         }
01101                         H5Dwrite(ds,H5T_NATIVE_USHORT,spc,spc,H5P_DEFAULT,usdata);
01102                         if(usdata) {delete [] usdata; usdata=0;}
01103                         break;
01104                 case EMUtil::EM_UCHAR:
01105                         cdata = new unsigned char[size];
01106                         for (size_t i = 0; i < size; ++i) {
01107                                 if(data[i] <= rendermin) {
01108                                         cdata[i] = 0;
01109                                 }
01110                                 else if(data[i] >= rendermax){
01111                                         cdata[i] = UCHAR_MAX;
01112                                 }
01113                                 else {
01114                                         cdata[i]=(unsigned char)((data[i]-rendermin)/(rendermax-rendermin)*UCHAR_MAX);
01115                                 }
01116                         }
01117                         H5Dwrite(ds,H5T_NATIVE_UCHAR,spc,spc,H5P_DEFAULT,cdata);
01118                         if(cdata) {delete [] cdata; cdata=0;}
01119                         break;
01120                 default:
01121                         throw ImageWriteException(filename,"HDF5 does not support this data format");
01122                 }
01123         }
01124 
01125         H5Sclose(spc);
01126         H5Dclose(ds);
01127         EXITFUNC;
01128         return 0;
01129 }
01130 
01131 int HdfIO2::get_nimg()
01132 {
01133         init();
01134         hid_t attr=H5Aopen_name(group,"imageid_max");
01135         int n = read_attr(attr);
01136         H5Aclose(attr);
01137 
01138         return n+1;
01139 }
01140 
01141 void HdfIO2::flush()
01142 {
01143         return;
01144 }
01145 
01146 bool HdfIO2::is_complex_mode()
01147 {
01148         return false;
01149 }
01150 
01151 // always big endian
01152 bool HdfIO2::is_image_big_endian()
01153 {
01154         return true;
01155 }
01156 
01157 
01158 
01159 #endif  //EM_HDF5

Generated on Thu Nov 17 12:43:45 2011 for EMAN2 by  doxygen 1.3.9.1