001 /* 002 * CDDL HEADER START 003 * 004 * The contents of this file are subject to the terms of the 005 * Common Development and Distribution License, Version 1.0 only 006 * (the "License"). You may not use this file except in compliance 007 * with the License. 008 * 009 * You can obtain a copy of the license at 010 * trunk/opends/resource/legal-notices/OpenDS.LICENSE 011 * or https://OpenDS.dev.java.net/OpenDS.LICENSE. 012 * See the License for the specific language governing permissions 013 * and limitations under the License. 014 * 015 * When distributing Covered Code, include this CDDL HEADER in each 016 * file and include the License file at 017 * trunk/opends/resource/legal-notices/OpenDS.LICENSE. If applicable, 018 * add the following below this CDDL HEADER, with the fields enclosed 019 * by brackets "[]" replaced with your own identifying information: 020 * Portions Copyright [yyyy] [name of copyright owner] 021 * 022 * CDDL HEADER END 023 * 024 * 025 * Copyright 2006-2008 Sun Microsystems, Inc. 026 */ 027 package org.opends.server.backends.jeb.importLDIF; 028 029 import org.opends.server.types.DN; 030 import org.opends.server.types.LDIFImportConfig; 031 import org.opends.server.types.AttributeType; 032 import org.opends.server.util.LDIFReader; 033 import org.opends.server.admin.std.server.LocalDBBackendCfg; 034 import org.opends.server.backends.jeb.*; 035 036 import com.sleepycat.je.DatabaseException; 037 import com.sleepycat.je.Transaction; 038 import com.sleepycat.je.LockMode; 039 040 import java.util.concurrent.BlockingQueue; 041 import java.util.concurrent.ConcurrentHashMap; 042 import java.util.concurrent.atomic.AtomicLong; 043 import java.util.*; 044 045 /** 046 * This class represents the import context for a destination base DN. 047 */ 048 public class DNContext { 049 050 /** 051 * The destination base DN. 052 */ 053 private DN baseDN; 054 055 /** 056 * The include branches below the base DN. 057 */ 058 private List<DN> includeBranches; 059 060 /** 061 * The exclude branches below the base DN. 062 */ 063 private List<DN> excludeBranches; 064 065 /** 066 * The configuration of the destination backend. 067 */ 068 private LocalDBBackendCfg config; 069 070 /** 071 * The requested LDIF import configuration. 072 */ 073 private LDIFImportConfig ldifImportConfig; 074 075 /** 076 * A reader for the source LDIF file. 077 */ 078 private LDIFReader ldifReader; 079 080 /** 081 * The entry entryContainer for the destination base DN. 082 */ 083 private EntryContainer entryContainer; 084 085 /** 086 * The source entryContainer if this is a partial import of a base DN. 087 */ 088 private EntryContainer srcEntryContainer; 089 090 /** 091 * A queue of elements that have been read from the LDIF and are ready 092 * to be imported. 093 */ 094 095 private BlockingQueue<WorkElement> workQueue; 096 097 098 //This currently isn't used. 099 private ArrayList<VLVIndex> vlvIndexes = new ArrayList<VLVIndex>(); 100 101 /** 102 * The maximum number of parent ID values that we will remember. 103 */ 104 private static final int PARENT_ID_MAP_SIZE = 100; 105 106 /** 107 * Map of likely parent entry DNs to their entry IDs. 108 */ 109 private HashMap<DN,EntryID> parentIDMap = 110 new HashMap<DN,EntryID>(PARENT_ID_MAP_SIZE); 111 112 //Map of pending DNs added to the work queue. Used to check if a parent 113 //entry has been added, but isn't in the dn2id DB. 114 private ConcurrentHashMap<DN,DN> pendingMap = 115 new ConcurrentHashMap<DN, DN>() ; 116 117 //Used to synchronize the parent ID map, since multiple worker threads 118 //can be accessing it. 119 private Object synchObject = new Object(); 120 121 /** 122 * The number of LDAP entries added to the database, used to update the 123 * entry database record count after import. The value is not updated 124 * for replaced entries. Multiple threads may be updating this value. 125 */ 126 private AtomicLong entryInsertCount = new AtomicLong(0); 127 128 /** 129 * The parent DN of the previous imported entry. 130 */ 131 private DN parentDN; 132 133 /** 134 * The superior IDs, in order from the parent up to the base DN, of the 135 * previous imported entry. This is used together with the previous parent DN 136 * to save time constructing the subtree index, in the typical case where many 137 * contiguous entries from the LDIF file have the same parent. 138 */ 139 private ArrayList<EntryID> IDs; 140 141 //The buffer manager used to hold the substring cache. 142 private BufferManager bufferManager; 143 144 145 /** 146 * Get the work queue. 147 * 148 * @return The work queue. 149 */ 150 public BlockingQueue<WorkElement> getWorkQueue() { 151 return workQueue; 152 } 153 154 155 /** 156 * Set the work queue to the specified work queue. 157 * 158 * @param workQueue The work queue. 159 */ 160 public void 161 setWorkQueue(BlockingQueue<WorkElement> workQueue) { 162 this.workQueue = workQueue; 163 } 164 165 /** 166 * Set the destination base DN. 167 * @param baseDN The destination base DN. 168 */ 169 public void setBaseDN(DN baseDN) 170 { 171 this.baseDN = baseDN; 172 } 173 174 /** 175 * Get the destination base DN. 176 * @return The destination base DN. 177 */ 178 public DN getBaseDN() 179 { 180 return baseDN; 181 } 182 183 /** 184 * Set the configuration of the destination backend. 185 * @param config The destination backend configuration. 186 */ 187 public void setConfig(LocalDBBackendCfg config) 188 { 189 this.config = config; 190 } 191 192 /** 193 * Get the configuration of the destination backend. 194 * @return The destination backend configuration. 195 */ 196 public LocalDBBackendCfg getConfig() 197 { 198 return config; 199 } 200 201 /** 202 * Set the requested LDIF import configuration. 203 * @param ldifImportConfig The LDIF import configuration. 204 */ 205 public void setLDIFImportConfig(LDIFImportConfig ldifImportConfig) 206 { 207 this.ldifImportConfig = ldifImportConfig; 208 } 209 210 /** 211 * Get the requested LDIF import configuration. 212 * @return The requested LDIF import configuration. 213 */ 214 public LDIFImportConfig getLDIFImportConfig() 215 { 216 return ldifImportConfig; 217 } 218 219 /** 220 * Set the source LDIF reader. 221 * @param ldifReader The source LDIF reader. 222 */ 223 public void setLDIFReader(LDIFReader ldifReader) 224 { 225 this.ldifReader = ldifReader; 226 } 227 228 /** 229 * Get the source LDIF reader. 230 * @return The source LDIF reader. 231 */ 232 public LDIFReader getLDIFReader() 233 { 234 return ldifReader; 235 } 236 237 /** 238 * Set the entry entryContainer for the destination base DN. 239 * @param entryContainer The entry entryContainer for the destination base DN. 240 */ 241 public void setEntryContainer(EntryContainer entryContainer) 242 { 243 this.entryContainer = entryContainer; 244 } 245 246 /** 247 * Get the entry entryContainer for the destination base DN. 248 * @return The entry entryContainer for the destination base DN. 249 */ 250 public EntryContainer getEntryContainer() 251 { 252 return entryContainer; 253 } 254 255 /** 256 * Set the source entry entryContainer for the destination base DN. 257 * @param srcEntryContainer The entry source entryContainer for the 258 * destination base DN. 259 */ 260 public void setSrcEntryContainer(EntryContainer srcEntryContainer) 261 { 262 this.srcEntryContainer = srcEntryContainer; 263 } 264 265 /** 266 * Get the source entry entryContainer for the destination base DN. 267 * @return The source entry entryContainer for the destination base DN. 268 */ 269 public EntryContainer getSrcEntryContainer() 270 { 271 return srcEntryContainer; 272 } 273 274 /** 275 * Get the number of new LDAP entries imported into the entry database. 276 * @return The number of new LDAP entries imported into the entry database. 277 */ 278 public long getEntryInsertCount() 279 { 280 return entryInsertCount.get(); 281 } 282 283 /** 284 * Increment the number of new LDAP entries imported into the entry database 285 * by the given amount. 286 * @param delta The amount to add. 287 */ 288 public void incrEntryInsertCount(long delta) 289 { 290 entryInsertCount.getAndAdd(delta); 291 } 292 293 /** 294 * Get the parent DN of the previous imported entry. 295 * @return The parent DN of the previous imported entry. 296 */ 297 public DN getParentDN() 298 { 299 return parentDN; 300 } 301 302 /** 303 * Set the parent DN of the previous imported entry. 304 * @param parentDN The parent DN of the previous imported entry. 305 */ 306 public void setParentDN(DN parentDN) 307 { 308 this.parentDN = parentDN; 309 } 310 311 /** 312 * Get the superior IDs of the previous imported entry. 313 * @return The superior IDs of the previous imported entry. 314 */ 315 public ArrayList<EntryID> getIDs() 316 { 317 return IDs; 318 } 319 320 /** 321 * Set the superior IDs of the previous imported entry. 322 * @param IDs The superior IDs of the previous imported entry. 323 */ 324 public void setIDs(ArrayList<EntryID> IDs) 325 { 326 this.IDs = IDs; 327 } 328 329 /** 330 * Retrieves the set of base DNs that specify the set of entries to 331 * exclude from the import. The contents of the returned list may 332 * be altered by the caller. 333 * 334 * @return The set of base DNs that specify the set of entries to 335 * exclude from the import. 336 */ 337 public List<DN> getExcludeBranches() 338 { 339 return excludeBranches; 340 } 341 342 343 344 /** 345 * Specifies the set of base DNs that specify the set of entries to 346 * exclude from the import. 347 * 348 * @param excludeBranches The set of base DNs that specify the set 349 * of entries to exclude from the import. 350 */ 351 public void setExcludeBranches(List<DN> excludeBranches) 352 { 353 if (excludeBranches == null) 354 { 355 this.excludeBranches = new ArrayList<DN>(0); 356 } 357 else 358 { 359 this.excludeBranches = excludeBranches; 360 } 361 } 362 363 364 365 /** 366 * Retrieves the set of base DNs that specify the set of entries to 367 * include in the import. The contents of the returned list may be 368 * altered by the caller. 369 * 370 * @return The set of base DNs that specify the set of entries to 371 * include in the import. 372 */ 373 public List<DN> getIncludeBranches() 374 { 375 return includeBranches; 376 } 377 378 379 380 /** 381 * Specifies the set of base DNs that specify the set of entries to 382 * include in the import. 383 * 384 * @param includeBranches The set of base DNs that specify the set 385 * of entries to include in the import. 386 */ 387 public void setIncludeBranches(List<DN> includeBranches) 388 { 389 if (includeBranches == null) 390 { 391 this.includeBranches = new ArrayList<DN>(0); 392 } 393 else 394 { 395 this.includeBranches = includeBranches; 396 } 397 } 398 399 400 /** 401 * Return the attribute type attribute index map. 402 * 403 * @return The attribute type attribute index map. 404 */ 405 public Map<AttributeType, AttributeIndex> getAttrIndexMap() { 406 return entryContainer.getAttributeIndexMap(); 407 } 408 409 /** 410 * Set all the indexes to trusted. 411 * 412 * @throws DatabaseException If the trusted value cannot be updated in the 413 * index DB. 414 */ 415 public void setIndexesTrusted() throws DatabaseException { 416 entryContainer.getID2Children().setTrusted(null,true); 417 entryContainer.getID2Subtree().setTrusted(null, true); 418 for(AttributeIndex attributeIndex : 419 entryContainer.getAttributeIndexes()) { 420 Index index; 421 if((index = attributeIndex.getEqualityIndex()) != null) { 422 index.setTrusted(null, true); 423 } 424 if((index=attributeIndex.getPresenceIndex()) != null) { 425 index.setTrusted(null, true); 426 } 427 if((index=attributeIndex.getSubstringIndex()) != null) { 428 index.setTrusted(null, true); 429 } 430 if((index=attributeIndex.getOrderingIndex()) != null) { 431 index.setTrusted(null, true); 432 } 433 if((index=attributeIndex.getApproximateIndex()) != null) { 434 index.setTrusted(null, true); 435 } 436 } 437 } 438 439 440 /** 441 * Get the Entry ID of the parent entry. 442 * @param parentDN The parent DN. 443 * @param dn2id The DN2ID DB. 444 * @param txn A database transaction, 445 * @return The entry ID of the parent entry. 446 * @throws DatabaseException If a DB error occurs. 447 */ 448 public 449 EntryID getParentID(DN parentDN, DN2ID dn2id, Transaction txn) 450 throws DatabaseException { 451 EntryID parentID; 452 synchronized(synchObject) { 453 parentID = parentIDMap.get(parentDN); 454 if (parentID != null) { 455 return parentID; 456 } 457 } 458 int i=0; 459 //If the parent is in the pending map, another thread is working on the 460 //parent entry; wait until that thread is done with the parent. 461 while(isPending(parentDN)) { 462 try { 463 Thread.sleep(50); 464 if(i == 3) { 465 return null; 466 } 467 i++; 468 } catch (Exception e) { 469 return null; 470 } 471 } 472 parentID = dn2id.get(txn, parentDN, LockMode.DEFAULT); 473 //If the parent is in dn2id, add it to the cache. 474 if (parentID != null) { 475 synchronized(synchObject) { 476 if (parentIDMap.size() >= PARENT_ID_MAP_SIZE) { 477 Iterator<DN> iterator = parentIDMap.keySet().iterator(); 478 iterator.next(); 479 iterator.remove(); 480 } 481 parentIDMap.put(parentDN, parentID); 482 } 483 } 484 return parentID; 485 } 486 487 /** 488 * Check if the parent DN is in the pending map. 489 * 490 * @param parentDN The DN of the parent. 491 * @return <CODE>True</CODE> if the parent is in the pending map. 492 */ 493 private boolean isPending(DN parentDN) { 494 boolean ret = false; 495 if(pendingMap.containsKey(parentDN)) { 496 ret = true; 497 } 498 return ret; 499 } 500 501 /** 502 * Add specified DN to the pending map. 503 * 504 * @param dn The DN to add to the map. 505 */ 506 public void addPending(DN dn) { 507 pendingMap.putIfAbsent(dn, dn); 508 } 509 510 /** 511 * Remove the specified DN from the pending map. 512 * 513 * @param dn The DN to remove from the map. 514 */ 515 public void removePending(DN dn) { 516 pendingMap.remove(dn); 517 } 518 519 /** 520 * Set the substring buffer manager to the specified buffer manager. 521 * 522 * @param bufferManager The buffer manager. 523 */ 524 public void setBufferManager(BufferManager bufferManager) { 525 this.bufferManager = bufferManager; 526 } 527 528 /** 529 * Return the buffer manager. 530 * 531 * @return The buffer manager. 532 */ 533 public BufferManager getBufferManager() { 534 return bufferManager; 535 } 536 }