Skip to content

Commit ed7e34f

Browse files
gorhillgorhill
committed
Refactor selfie generation into a more flexible persistence mechanism
The motivation is to address the higher peak memory usage at launch time with 3rd-gen HNTrie when a selfie was present. The selfie generation prior to this change was to collect all filtering data into a single data structure, and then to serialize that whole structure at once into storage (using JSON.stringify). However, HNTrie serialization requires that a large UintArray32 be converted into a plain JS array, which itslef would be indirectly converted into a JSON string. This was the main reason why peak memory usage would be higher at launch from selfie, since the JSON string would need to be wholly unserialized into JS objects, which themselves would need to be converted into more specialized data structures (like that Uint32Array one). The solution to lower peak memory usage at launch is to refactor selfie generation to allow a more piecemeal approach: each filtering component is given the ability to serialize itself rather than to be forced to be embedded in the master selfie. With this approach, the HNTrie buffer can now serialize to its own storage by converting the buffer data directly into a string which can be directly sent to storage. This avoiding expensive intermediate steps such as converting into a JS array and then to a JSON string. As part of the refactoring, there was also opportunistic code upgrade to ES6 and Promise (eventually all of uBO's code will be proper ES6). Additionally, the polyfill to bring getBytesInUse() to Firefox has been revisited to replace the rather expensive previous implementation with an implementation with virtually no overhead.
1 parent 83a3767 commit ed7e34f

File tree

14 files changed

+585
-319
lines changed

14 files changed

+585
-319
lines changed

‎.jshintrc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"browser": false, // global variable in Firefox, Edge
88
"chrome": false, // global variable in Chromium, Chrome, Opera
99
"Components": false, // global variable in Firefox
10+
"log": false,
1011
"safari": false,
1112
"self": false,
1213
"vAPI": false,

‎platform/chromium/vapi-common.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@
3030

3131
/******************************************************************************/
3232

33+
vAPI.T0 = Date.now();
34+
35+
/******************************************************************************/
36+
3337
vAPI.setTimeout = vAPI.setTimeout || self.setTimeout.bind(self);
3438

3539
/******************************************************************************/

‎src/background.html‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
<title>uBlock Origin</title>
66
</head>
77
<body>
8+
<script src="js/console.js"></script>
89
<script src="lib/lz4/lz4-block-codec-any.js"></script>
910
<script src="lib/punycode.js"></script>
1011
<script src="lib/publicsuffixlist/publicsuffixlist.js"></script>

‎src/js/assets.js‎

Lines changed: 89 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -449,26 +449,22 @@ const assetCacheRegistryStartTime = Date.now();
449449
let assetCacheRegistryPromise;
450450
let assetCacheRegistry = {};
451451

452-
const getAssetCacheRegistry = function(callback) {
452+
const getAssetCacheRegistry = function() {
453453
if ( assetCacheRegistryPromise === undefined ) {
454454
assetCacheRegistryPromise = new Promise(resolve => {
455-
// start of executor
456-
µBlock.cacheStorage.get('assetCacheRegistry', bin => {
457-
if (
458-
bin instanceof Object &&
459-
bin.assetCacheRegistry instanceof Object
460-
) {
461-
assetCacheRegistry = bin.assetCacheRegistry;
462-
}
463-
resolve();
464-
});
465-
// end of executor
455+
µBlock.cacheStorage.get('assetCacheRegistry', bin => {
456+
if (
457+
bin instanceof Object &&
458+
bin.assetCacheRegistry instanceof Object
459+
) {
460+
assetCacheRegistry = bin.assetCacheRegistry;
461+
}
462+
resolve();
463+
});
466464
});
467465
}
468466

469-
assetCacheRegistryPromise.then(( ) => {
470-
callback(assetCacheRegistry);
471-
});
467+
return assetCacheRegistryPromise.then(( ) => assetCacheRegistry);
472468
};
473469

474470
const saveAssetCacheRegistry = (function() {
@@ -513,11 +509,9 @@ const assetCacheRead = function(assetKey, callback) {
513509
reportBack(bin[internalKey]);
514510
};
515511

516-
let onReady = function() {
512+
getAssetCacheRegistry().then(( ) => {
517513
µBlock.cacheStorage.get(internalKey, onAssetRead);
518-
};
519-
520-
getAssetCacheRegistry(onReady);
514+
});
521515
};
522516

523517
const assetCacheWrite = function(assetKey, details, callback) {
@@ -542,22 +536,35 @@ const assetCacheWrite = function(assetKey, details, callback) {
542536
if ( details instanceof Object && typeof details.url === 'string' ) {
543537
entry.remoteURL = details.url;
544538
}
545-
µBlock.cacheStorage.set({ assetCacheRegistry, [internalKey]: content });
539+
µBlock.cacheStorage.set(
540+
{ [internalKey]: content },
541+
details => {
542+
if (
543+
details instanceof Object &&
544+
typeof details.bytesInUse === 'number'
545+
) {
546+
entry.byteLength = details.bytesInUse;
547+
}
548+
saveAssetCacheRegistry(true);
549+
}
550+
);
546551
const result = { assetKey, content };
547552
if ( typeof callback === 'function' ) {
548553
callback(result);
549554
}
550555
// https://github.com/uBlockOrigin/uBlock-issues/issues/248
551556
fireNotification('after-asset-updated', result);
552557
};
553-
getAssetCacheRegistry(onReady);
558+
559+
getAssetCacheRegistry().then(( ) => {
560+
µBlock.cacheStorage.get(internalKey, onReady);
561+
});
554562
};
555563

556564
const assetCacheRemove = function(pattern, callback) {
557-
const onReady = function() {
558-
const cacheDict = assetCacheRegistry,
559-
removedEntries = [],
560-
removedContent = [];
565+
getAssetCacheRegistry().then(cacheDict => {
566+
const removedEntries = [];
567+
const removedContent = [];
561568
for ( const assetKey in cacheDict ) {
562569
if ( pattern instanceof RegExp && !pattern.test(assetKey) ) {
563570
continue;
@@ -582,14 +589,15 @@ const assetCacheRemove = function(pattern, callback) {
582589
{ assetKey: removedEntries[i] }
583590
);
584591
}
585-
};
586-
587-
getAssetCacheRegistry(onReady);
592+
});
588593
};
589594

590595
const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
591-
const onReady = function() {
592-
const cacheDict = assetCacheRegistry;
596+
if ( typeof exclude === 'function' ) {
597+
callback = exclude;
598+
exclude = undefined;
599+
}
600+
getAssetCacheRegistry().then(cacheDict => {
593601
let mustSave = false;
594602
for ( const assetKey in cacheDict ) {
595603
if ( pattern instanceof RegExp ) {
@@ -617,12 +625,7 @@ const assetCacheMarkAsDirty = function(pattern, exclude, callback) {
617625
if ( typeof callback === 'function' ) {
618626
callback();
619627
}
620-
};
621-
if ( typeof exclude === 'function' ) {
622-
callback = exclude;
623-
exclude = undefined;
624-
}
625-
getAssetCacheRegistry(onReady);
628+
});
626629
};
627630

628631
/******************************************************************************/
@@ -642,12 +645,12 @@ const stringIsNotEmpty = function(s) {
642645
643646
**/
644647

645-
var readUserAsset = function(assetKey, callback) {
646-
var reportBack = function(content) {
648+
const readUserAsset = function(assetKey, callback) {
649+
const reportBack = function(content) {
647650
callback({ assetKey: assetKey, content: content });
648651
};
649652

650-
var onLoaded = function(bin) {
653+
const onLoaded = function(bin) {
651654
if ( !bin ) { return reportBack(''); }
652655
var content = '';
653656
if ( typeof bin['cached_asset_content://assets/user/filters.txt'] === 'string' ) {
@@ -671,7 +674,7 @@ var readUserAsset = function(assetKey, callback) {
671674
}
672675
return reportBack(content);
673676
};
674-
var toRead = assetKey;
677+
let toRead = assetKey;
675678
if ( assetKey === µBlock.userFiltersPath ) {
676679
toRead = [
677680
assetKey,
@@ -682,7 +685,7 @@ var readUserAsset = function(assetKey, callback) {
682685
vAPI.storage.get(toRead, onLoaded);
683686
};
684687

685-
var saveUserAsset = function(assetKey, content, callback) {
688+
const saveUserAsset = function(assetKey, content, callback) {
686689
var bin = {};
687690
bin[assetKey] = content;
688691
// TODO(seamless migration):
@@ -711,27 +714,33 @@ api.get = function(assetKey, options, callback) {
711714
callback = noopfunc;
712715
}
713716

717+
return new Promise(resolve => {
718+
// start of executor
714719
if ( assetKey === µBlock.userFiltersPath ) {
715-
readUserAsset(assetKey, callback);
720+
readUserAsset(assetKey, details => {
721+
callback(details);
722+
resolve(details);
723+
});
716724
return;
717725
}
718726

719-
var assetDetails = {},
727+
let assetDetails = {},
720728
contentURLs,
721729
contentURL;
722730

723-
var reportBack = function(content, err) {
724-
var details = { assetKey: assetKey, content: content };
731+
const reportBack = function(content, err) {
732+
const details = { assetKey: assetKey, content: content };
725733
if ( err ) {
726734
details.error = assetDetails.lastError = err;
727735
} else {
728736
assetDetails.lastError = undefined;
729737
}
730738
callback(details);
739+
resolve(details);
731740
};
732741

733-
var onContentNotLoaded = function() {
734-
var isExternal;
742+
const onContentNotLoaded = function() {
743+
let isExternal;
735744
while ( (contentURL = contentURLs.shift()) ) {
736745
isExternal = reIsExternalPath.test(contentURL);
737746
if ( isExternal === false || assetDetails.hasLocalURL !== true ) {
@@ -748,7 +757,7 @@ api.get = function(assetKey, options, callback) {
748757
}
749758
};
750759

751-
var onContentLoaded = function(details) {
760+
const onContentLoaded = function(details) {
752761
if ( stringIsNotEmpty(details.content) === false ) {
753762
onContentNotLoaded();
754763
return;
@@ -762,7 +771,7 @@ api.get = function(assetKey, options, callback) {
762771
reportBack(details.content);
763772
};
764773

765-
var onCachedContentLoaded = function(details) {
774+
const onCachedContentLoaded = function(details) {
766775
if ( details.content !== '' ) {
767776
return reportBack(details.content);
768777
}
@@ -780,11 +789,13 @@ api.get = function(assetKey, options, callback) {
780789
};
781790

782791
assetCacheRead(assetKey, onCachedContentLoaded);
792+
// end of executor
793+
});
783794
};
784795

785796
/******************************************************************************/
786797

787-
var getRemote = function(assetKey, callback) {
798+
const getRemote = function(assetKey, callback) {
788799
var assetDetails = {},
789800
contentURLs,
790801
contentURL;
@@ -852,10 +863,19 @@ var getRemote = function(assetKey, callback) {
852863
/******************************************************************************/
853864

854865
api.put = function(assetKey, content, callback) {
855-
if ( reIsUserAsset.test(assetKey) ) {
856-
return saveUserAsset(assetKey, content, callback);
857-
}
858-
assetCacheWrite(assetKey, content, callback);
866+
return new Promise(resolve => {
867+
const onDone = function(details) {
868+
if ( typeof callback === 'function' ) {
869+
callback(details);
870+
}
871+
resolve(details);
872+
};
873+
if ( reIsUserAsset.test(assetKey) ) {
874+
saveUserAsset(assetKey, content, onDone);
875+
} else {
876+
assetCacheWrite(assetKey, content, onDone);
877+
}
878+
});
859879
};
860880

861881
/******************************************************************************/
@@ -895,14 +915,27 @@ api.metadata = function(callback) {
895915
if ( cacheRegistryReady ) { onReady(); }
896916
});
897917

898-
getAssetCacheRegistry(function() {
918+
getAssetCacheRegistry().then(( ) => {
899919
cacheRegistryReady = true;
900920
if ( assetRegistryReady ) { onReady(); }
901921
});
902922
};
903923

904924
/******************************************************************************/
905925

926+
api.getBytesInUse = function() {
927+
return getAssetCacheRegistry().then(cacheDict => {
928+
let bytesUsed = 0;
929+
for ( const assetKey in cacheDict ) {
930+
if ( cacheDict.hasOwnProperty(assetKey) === false ) { continue; }
931+
bytesUsed += cacheDict[assetKey].byteLength || 0;
932+
}
933+
return bytesUsed;
934+
});
935+
};
936+
937+
/******************************************************************************/
938+
906939
api.purge = assetCacheMarkAsDirty;
907940

908941
api.remove = function(pattern, callback) {
@@ -1013,7 +1046,7 @@ var updateNext = function() {
10131046
updateOne();
10141047
});
10151048

1016-
getAssetCacheRegistry(function(dict) {
1049+
getAssetCacheRegistry().then(dict => {
10171050
cacheDict = dict;
10181051
if ( !assetDict ) { return; }
10191052
updateOne();

‎src/js/background.js‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,15 @@ const µBlock = (function() { // jshint ignore:line
4646
cacheStorageAPI: 'unset',
4747
cacheStorageCompression: true,
4848
cacheControlForFirefox1376932: 'no-cache, no-store, must-revalidate',
49+
consoleLogLevel: 'unset',
4950
debugScriptlets: false,
5051
disableWebAssembly: false,
5152
ignoreRedirectFilters: false,
5253
ignoreScriptInjectFilters: false,
5354
manualUpdateAssetFetchPeriod: 500,
5455
popupFontSize: 'unset',
5556
requestJournalProcessPeriod: 1000,
57+
selfieAfter: 11,
5658
strictBlockingBypassDuration: 120,
5759
suspendTabsUntilReady: false,
5860
userResourcesLocation: 'unset'
@@ -95,13 +97,13 @@ const µBlock = (function() { // jshint ignore:line
9597

9698
hiddenSettingsDefault: hiddenSettingsDefault,
9799
hiddenSettings: (function() {
98-
let out = Object.assign({}, hiddenSettingsDefault),
100+
const out = Object.assign({}, hiddenSettingsDefault),
99101
json = vAPI.localStorage.getItem('immediateHiddenSettings');
100102
if ( typeof json === 'string' ) {
101103
try {
102-
let o = JSON.parse(json);
104+
const o = JSON.parse(json);
103105
if ( o instanceof Object ) {
104-
for ( let k in o ) {
106+
for ( const k in o ) {
105107
if ( out.hasOwnProperty(k) ) {
106108
out[k] = o[k];
107109
}
@@ -111,8 +113,6 @@ const µBlock = (function() { // jshint ignore:line
111113
catch(ex) {
112114
}
113115
}
114-
// Remove once 1.15.12+ is widespread.
115-
vAPI.localStorage.removeItem('hiddenSettings');
116116
return out;
117117
})(),
118118

@@ -138,7 +138,7 @@ const µBlock = (function() { // jshint ignore:line
138138
// Read-only
139139
systemSettings: {
140140
compiledMagic: 6, // Increase when compiled format changes
141-
selfieMagic: 7 // Increase when selfie format changes
141+
selfieMagic: 8 // Increase when selfie format changes
142142
},
143143

144144
restoreBackupSettings: {
@@ -161,8 +161,6 @@ const µBlock = (function() { // jshint ignore:line
161161
selectedFilterLists: [],
162162
availableFilterLists: {},
163163

164-
selfieAfter: 17 * oneMinute,
165-
166164
pageStores: new Map(),
167165
pageStoresToken: 0,
168166

0 commit comments

Comments
 (0)