Ptex
PtexCache.cpp
Go to the documentation of this file.
1 /*
2 PTEX SOFTWARE
3 Copyright 2014 Disney Enterprises, Inc. All rights reserved
4 
5 Redistribution and use in source and binary forms, with or without
6 modification, are permitted provided that the following conditions are
7 met:
8 
9  * Redistributions of source code must retain the above copyright
10  notice, this list of conditions and the following disclaimer.
11 
12  * Redistributions in binary form must reproduce the above copyright
13  notice, this list of conditions and the following disclaimer in
14  the documentation and/or other materials provided with the
15  distribution.
16 
17  * The names "Disney", "Walt Disney Pictures", "Walt Disney Animation
18  Studios" or the names of its contributors may NOT be used to
19  endorse or promote products derived from this software without
20  specific prior written permission from Walt Disney Pictures.
21 
22 Disclaimer: THIS SOFTWARE IS PROVIDED BY WALT DISNEY PICTURES AND
23 CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
24 BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS
25 FOR A PARTICULAR PURPOSE, NONINFRINGEMENT AND TITLE ARE DISCLAIMED.
26 IN NO EVENT SHALL WALT DISNEY PICTURES, THE COPYRIGHT HOLDER OR
27 CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
28 EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
29 PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
30 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND BASED ON ANY
31 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
34 */
35 
69 #include "PtexPlatform.h"
70 #include <algorithm>
71 #include <sys/types.h>
72 #include <sys/stat.h>
73 #include <stdlib.h>
74 #include <iostream>
75 #include <ctype.h>
76 #include "Ptexture.h"
77 #include "PtexReader.h"
78 #include "PtexCache.h"
79 
80 
82 
84 {
85  if (0 == unref()) {
86  _cache->logRecentlyUsed(this);
87  }
88 }
89 
90 
91 bool PtexReaderCache::findFile(const char*& filename, std::string& buffer, Ptex::String& error)
92 {
93  bool isAbsolute = (filename[0] == '/'
94 #ifdef PTEX_PLATFORM_WINDOWS
95  || filename[0] == '\\'
96  || (isalpha(filename[0]) && filename[1] == ':')
97 #endif
98  );
99  if (isAbsolute || _searchdirs.empty()) return true; // no need to search
100 
101  // file is relative, search in searchpath
102  buffer.reserve(256); // minimize reallocs (will grow automatically)
103  struct stat statbuf;
104  for (size_t i = 0, size = _searchdirs.size(); i < size; i++) {
105  buffer = _searchdirs[i];
106  buffer += "/";
107  buffer += filename;
108  if (stat(buffer.c_str(), &statbuf) == 0) {
109  filename = buffer.c_str();
110  return true;
111  }
112  }
113  // not found
114  std::string errstr = "Can't find ptex file: ";
115  errstr += filename;
116  error = errstr.c_str();
117  return false;
118 }
119 
120 
121 PtexTexture* PtexReaderCache::get(const char* filename, Ptex::String& error)
122 {
123  // lookup reader in map
124  StringKey key(filename);
125  PtexCachedReader* reader = _files.get(key);
126  bool isNew = false;
127 
128  if (reader) {
129  if (!reader->ok()) return 0;
130  if (reader->pendingPurge()) {
131  // a previous purge attempt was made and file was busy. Try again now.
132  purge(reader);
133  }
134  reader->ref();
135  } else {
136  reader = new PtexCachedReader(_premultiply, _io, _err, this);
137  isNew = true;
138  }
139 
140  bool needOpen = reader->needToOpen();
141  if (needOpen) {
142  std::string buffer;
143  const char* pathToOpen = filename;
144  // search for the file (unless we have an I/O handler)
145  if (_io || findFile(pathToOpen, buffer, error)) {
146  reader->open(pathToOpen, error);
147  } else {
148  // flag reader as invalid so we don't try to open it again on next lookup
149  reader->invalidate();
150  }
151  }
152 
153  if (isNew) {
154  size_t newMemUsed = 0;
155  PtexCachedReader* newreader = reader;
156  reader = _files.tryInsert(key, reader, newMemUsed);
157  adjustMemUsed(newMemUsed);
158  if (reader != newreader) {
159  // another thread got here first
160  reader->ref();
161  delete newreader;
162  }
163  }
164 
165  if (!reader->ok()) {
166  reader->unref();
167  return 0;
168  }
169 
170  if (needOpen) {
171  reader->logOpen();
172  }
173 
174  return reader;
175 }
176 
177 PtexCache* PtexCache::create(int maxFiles, size_t maxMem, bool premultiply,
178  PtexInputHandler* inputHandler,
179  PtexErrorHandler* errorHandler)
180 {
181  // set default files to 100
182  if (maxFiles <= 0) maxFiles = 100;
183 
184  return new PtexReaderCache(maxFiles, maxMem, premultiply, inputHandler, errorHandler);
185 }
186 
187 
189 {
190  while (1) {
191  MruList* mruList = _mruList;
192  int slot = AtomicIncrement(&mruList->next)-1;
193  if (slot < numMruFiles) {
194  mruList->files[slot] = reader;
195  return;
196  }
197  // no mru slot available, process mru list and try again
198  do processMru();
199  while (_mruList->next >= numMruFiles);
200  }
201 }
202 
204 {
205  // use a non-blocking lock so we can proceed as soon as space has been freed in the mru list
206  // (which happens almost immediately in the processMru thread that has the lock)
207  if (!_mruLock.trylock()) return;
208  if (_mruList->next < numMruFiles) {
209  _mruLock.unlock();
210  return;
211  }
212 
213  // switch mru buffers and reset slot counter so other threads can proceed immediately
214  MruList* mruList = _mruList;
216  _prevMruList = mruList;
217 
218  // extract relevant stats and add to open/active list
219  size_t memUsedChange = 0, filesOpenChange = 0;
220  for (int i = 0; i < numMruFiles; ++i) {
221  PtexCachedReader* reader;
222  do { reader = mruList->files[i]; } while (!reader); // loop on (unlikely) race condition
223  mruList->files[i] = 0;
224  memUsedChange += reader->getMemUsedChange();
225  size_t opens = reader->getOpensChange();
226  size_t blockReads = reader->getBlockReadsChange();
227  filesOpenChange += opens;
228  if (opens || blockReads) {
229  _fileOpens += opens;
230  _blockReads += blockReads;
231  _openFiles.push(reader);
232  }
233  if (_maxMem) {
234  _activeFiles.push(reader);
235  }
236  }
237  AtomicStore(&mruList->next, 0);
238  adjustMemUsed(memUsedChange);
239  adjustFilesOpen(filesOpenChange);
240 
241  bool shouldPruneFiles = _filesOpen > _maxFiles;
242  bool shouldPruneData = _maxMem && _memUsed > _maxMem;
243 
244  if (shouldPruneFiles) {
245  pruneFiles();
246  }
247  if (shouldPruneData) {
248  pruneData();
249  }
250  _mruLock.unlock();
251 }
252 
253 
255 {
256  size_t numToClose = _filesOpen - _maxFiles;
257  if (numToClose > 0) {
258  while (numToClose) {
259  PtexCachedReader* reader = _openFiles.pop();
260  if (!reader) { _filesOpen = 0; break; }
261  if (reader->tryClose()) {
262  --numToClose;
263  --_filesOpen;
264  }
265  }
266  }
267 }
268 
269 
271 {
272  size_t memUsedChangeTotal = 0;
273  size_t memUsed = _memUsed;
274  while (memUsed + memUsedChangeTotal > _maxMem) {
275  PtexCachedReader* reader = _activeFiles.pop();
276  if (!reader) break;
277  size_t memUsedChange;
278  if (reader->tryPrune(memUsedChange)) {
279  // Note: after clearing, memUsedChange is negative
280  memUsedChangeTotal += memUsedChange;
281  }
282  }
283  adjustMemUsed(memUsedChangeTotal);
284 }
285 
286 
288 {
289  PtexCachedReader* reader = static_cast<PtexCachedReader*>(texture);
290  reader->unref();
291  purge(reader);
292  reader->ref();
293 }
294 
295 
296 void PtexReaderCache::purge(const char* filename)
297 {
298  StringKey key(filename);
299  PtexCachedReader* reader = _files.get(key);
300  if (reader) purge(reader);
301 }
302 
304 {
305  size_t memUsedChange;
306  if (reader->tryPurge(memUsedChange)) {
307  adjustMemUsed(memUsedChange);
308  }
309 }
310 
312 {
313  size_t memUsedChange;
314  if (reader->tryPurge(memUsedChange)) {
315  memUsedChangeTotal += memUsedChange;
316  }
317 }
318 
320 {
321  Purger purger;
322  _files.foreach(purger);
324 }
325 
327 {
328  stats.memUsed = _memUsed;
329  stats.peakMemUsed = _peakMemUsed;
330  stats.filesOpen = _filesOpen;
332  stats.filesAccessed = _files.size();
333  stats.fileReopens = _fileOpens < stats.filesAccessed ? 0 : _fileOpens - stats.filesAccessed;
334  stats.blockReads = _blockReads;
335 }
336 
Platform-specific classes, functions, and includes.
PTEX_INLINE void AtomicStore(T volatile *target, T value)
Definition: PtexPlatform.h:284
PTEX_INLINE T AtomicIncrement(volatile T *target)
Definition: PtexPlatform.h:225
#define PTEX_NAMESPACE_END
Definition: PtexVersion.h:62
Public API classes for reading, writing, caching, and filtering Ptex files.
bool trylock()
Definition: PtexPlatform.h:142
void unlock()
Definition: PtexPlatform.h:143
File-handle and memory cache for reading ptex files.
Definition: Ptexture.h:684
static PtexCache * create(int maxFiles, size_t maxMem, bool premultiply=false, PtexInputHandler *inputHandler=0, PtexErrorHandler *errorHandler=0)
Create a cache with the specified limits.
Definition: PtexCache.cpp:177
PtexReaderCache * _cache
Definition: PtexCache.h:110
int32_t unref()
Definition: PtexCache.h:146
bool tryPurge(size_t &memUsedChange)
Definition: PtexCache.h:162
size_t getMemUsedChange()
Definition: PtexCache.h:173
bool tryPrune(size_t &memUsedChange)
Definition: PtexCache.h:152
virtual void release()
Release resources held by this pointer (pointer becomes invalid).
Definition: PtexCache.cpp:83
size_t getBlockReadsChange()
Definition: PtexCache.h:187
size_t getOpensChange()
Definition: PtexCache.h:180
Custom handler interface redirecting Ptex error messages.
Definition: Ptexture.h:658
void foreach(Fn &fn)
Definition: PtexHashMap.h:252
uint32_t size() const
Definition: PtexHashMap.h:201
Value get(Key &key)
Definition: PtexHashMap.h:203
Value tryInsert(Key &key, Value value, size_t &newMemUsed)
Definition: PtexHashMap.h:224
Custom handler interface for intercepting and redirecting Ptex input stream calls.
Definition: Ptexture.h:619
T * pop()
Definition: PtexCache.h:95
void push(T *node)
Definition: PtexCache.h:88
Cache for reading Ptex texture files.
Definition: PtexCache.h:198
PtexLruList< PtexCachedReader, &PtexCachedReader::_activeFilesItem > _activeFiles
Definition: PtexCache.h:301
PtexInputHandler * _io
Definition: PtexCache.h:280
size_t _maxMem
Definition: PtexCache.h:279
void adjustFilesOpen(size_t amount)
Definition: PtexCache.h:259
size_t _fileOpens
Definition: PtexCache.h:305
volatile size_t _memUsed
Definition: PtexCache.h:287
void logRecentlyUsed(PtexCachedReader *reader)
Definition: PtexCache.cpp:188
MruList *volatile _prevMruList
Definition: PtexCache.h:298
FileMap _files
Definition: PtexCache.h:285
void adjustMemUsed(size_t amount)
Definition: PtexCache.h:253
size_t _peakMemUsed
Definition: PtexCache.h:303
virtual PtexTexture * get(const char *path, Ptex::String &error)
Access a texture.
Definition: PtexCache.cpp:121
bool findFile(const char *&filename, std::string &buffer, Ptex::String &error)
Definition: PtexCache.cpp:91
virtual void getStats(Stats &stats)
Get stats.
Definition: PtexCache.cpp:326
static const int numMruFiles
Definition: PtexCache.h:291
size_t _maxFiles
Definition: PtexCache.h:278
size_t _peakFilesOpen
Definition: PtexCache.h:304
PtexErrorHandler * _err
Definition: PtexCache.h:281
PtexLruList< PtexCachedReader, &PtexCachedReader::_openFilesItem > _openFiles
Definition: PtexCache.h:300
virtual void purgeAll()
Remove all texture files from the cache.
Definition: PtexCache.cpp:319
size_t _blockReads
Definition: PtexCache.h:306
volatile size_t _filesOpen
Definition: PtexCache.h:288
MruList *volatile _mruList
Definition: PtexCache.h:297
virtual void purge(PtexTexture *)
Remove a texture file from the cache.
Definition: PtexCache.cpp:287
std::vector< std::string > _searchdirs
Definition: PtexCache.h:283
bool ok() const
Definition: PtexReader.h:64
bool needToOpen() const
Definition: PtexReader.h:57
bool pendingPurge() const
Definition: PtexReader.h:62
void invalidate()
Definition: PtexReader.h:66
void logOpen()
Definition: PtexReader.h:72
bool open(const char *path, Ptex::String &error)
Definition: PtexReader.cpp:137
bool tryClose()
Definition: PtexReader.cpp:219
Interface for reading data from a ptex file.
Definition: Ptexture.h:457
Memory-managed string.
Definition: Ptexture.h:296
const char * c_str() const
Definition: Ptexture.h:304
uint64_t filesAccessed
Definition: Ptexture.h:784
uint64_t filesOpen
Definition: Ptexture.h:782
uint64_t peakFilesOpen
Definition: Ptexture.h:783
uint64_t fileReopens
Definition: Ptexture.h:785
uint64_t memUsed
Definition: Ptexture.h:780
uint64_t blockReads
Definition: Ptexture.h:786
uint64_t peakMemUsed
Definition: Ptexture.h:781
PtexCachedReader *volatile files[numMruFiles]
Definition: PtexCache.h:294
void operator()(PtexCachedReader *reader)
Definition: PtexCache.cpp:311