Skip to content

Commit 068764b

Browse files
CarlChaoCarlCarlChaoCarl
authored
feature(db): optimize properties query (#5378)
1 parent ad728fa commit 068764b

6 files changed

Lines changed: 245 additions & 0 deletions

File tree

‎chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ public abstract class AbstractSnapshot<K, V> implements Snapshot {
1515

1616
protected WeakReference<Snapshot> next;
1717

18+
protected boolean isOptimized;
19+
1820
@Override
1921
public Snapshot advance() {
2022
return new SnapshotImpl(this);
@@ -34,4 +36,9 @@ public void setNext(Snapshot next) {
3436
public String getDbName() {
3537
return db.getDbName();
3638
}
39+
40+
@Override
41+
public boolean isOptimized(){
42+
return isOptimized;
43+
}
3744
}

‎chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,4 +46,8 @@ static boolean isImpl(Snapshot snapshot) {
4646
void updateSolidity();
4747

4848
String getDbName();
49+
50+
boolean isOptimized();
51+
52+
void reloadToMem();
4953
}

‎chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ public class SnapshotImpl extends AbstractSnapshot<Key, Value> {
3030
}
3131
previous = snapshot;
3232
snapshot.setNext(this);
33+
isOptimized = snapshot.isOptimized();
34+
if (isOptimized && root == previous) {
35+
Streams.stream(root.iterator()).forEach( e -> put(e.getKey(),e.getValue()));
36+
}
3337
}
3438

3539
@Override
@@ -40,6 +44,7 @@ public byte[] get(byte[] key) {
4044
private byte[] get(Snapshot head, byte[] key) {
4145
Snapshot snapshot = head;
4246
Value value;
47+
4348
while (Snapshot.isImpl(snapshot)) {
4449
if ((value = ((SnapshotImpl) snapshot).db.get(Key.of(key))) != null) {
4550
return value.getBytes();
@@ -83,6 +88,19 @@ public void merge(Snapshot from) {
8388
Streams.stream(fromImpl.db).forEach(e -> db.put(e.getKey(), e.getValue()));
8489
}
8590

91+
public void mergeAhead(Snapshot from) {
92+
if (from instanceof SnapshotRoot) {
93+
return ;
94+
}
95+
SnapshotImpl fromImpl = (SnapshotImpl) from;
96+
Streams.stream(fromImpl.db).forEach(e -> {
97+
if (db.get(e.getKey()) == null) {
98+
db.put(e.getKey(), e.getValue());
99+
}
100+
}
101+
);
102+
}
103+
86104
@Override
87105
public Snapshot retreat() {
88106
return previous;
@@ -177,4 +195,9 @@ public String getDbName() {
177195
public Snapshot newInstance() {
178196
return new SnapshotImpl(this);
179197
}
198+
199+
@Override
200+
public void reloadToMem() {
201+
mergeAhead(previous);
202+
}
180203
}

‎chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,12 @@ public synchronized void commit() {
221221
}
222222

223223
--activeSession;
224+
225+
dbs.forEach(db -> {
226+
if (db.getHead().isOptimized()) {
227+
db.getHead().reloadToMem();
228+
}
229+
});
224230
}
225231

226232
public synchronized void pop() {

‎chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public SnapshotRoot(DB<byte[], byte[]> db) {
3838
if (CACHE_DBS.contains(this.db.getDbName())) {
3939
this.cache = CacheManager.allocate(CacheType.findByType(this.db.getDbName()));
4040
}
41+
isOptimized = "properties".equalsIgnoreCase(db.getDbName());
4142
}
4243

4344
private boolean needOptAsset() {
@@ -221,4 +222,7 @@ public String getDbName() {
221222
public Snapshot newInstance() {
222223
return new SnapshotRoot(db.newInstance());
223224
}
225+
226+
@Override
227+
public void reloadToMem() { }
224228
}

‎framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
package org.tron.core.db2;
2+
3+
import static org.junit.Assert.assertEquals;
4+
import static org.junit.Assert.assertNull;
5+
6+
import java.io.File;
7+
import java.lang.reflect.Constructor;
8+
import org.junit.After;
9+
import org.junit.Assert;
10+
import org.junit.Before;
11+
import org.junit.Test;
12+
import org.tron.common.application.Application;
13+
import org.tron.common.application.ApplicationFactory;
14+
import org.tron.common.application.TronApplicationContext;
15+
import org.tron.common.utils.FileUtil;
16+
import org.tron.core.Constant;
17+
import org.tron.core.config.DefaultConfig;
18+
import org.tron.core.config.args.Args;
19+
import org.tron.core.db2.core.Snapshot;
20+
import org.tron.core.db2.core.SnapshotImpl;
21+
import org.tron.core.db2.core.SnapshotManager;
22+
import org.tron.core.db2.core.SnapshotRoot;
23+
24+
public class SnapshotImplTest {
25+
private RevokingDbWithCacheNewValueTest.TestRevokingTronStore tronDatabase;
26+
private TronApplicationContext context;
27+
private Application appT;
28+
private SnapshotManager revokingDatabase;
29+
30+
@Before
31+
public void init() {
32+
Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF);
33+
context = new TronApplicationContext(DefaultConfig.class);
34+
appT = ApplicationFactory.create(context);
35+
36+
tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore(
37+
"testSnapshotRoot-testMerge");
38+
revokingDatabase = context.getBean(SnapshotManager.class);
39+
revokingDatabase.enable();
40+
revokingDatabase.add(tronDatabase.getRevokingDB());
41+
}
42+
43+
@After
44+
public void removeDb() {
45+
Args.clearParam();
46+
context.destroy();
47+
FileUtil.deleteDir(new File("output_revokingStore_test"));
48+
49+
tronDatabase.close();
50+
revokingDatabase.shutdown();
51+
}
52+
53+
/**
54+
* linklist is: from -> root
55+
* root:key1=>value1, key2=>value2
56+
* from:key3=>value3, key4=>value4
57+
* after construct, getSnapshotImplIns(root);
58+
* from: key1=>value1, key2=>value2, key3=>value3, key4=>value4
59+
* from: get key1 or key2, traverse 0 times
60+
*/
61+
@Test
62+
public void testMergeRoot() {
63+
// linklist is: from -> root
64+
SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb());
65+
//root.setOptimized(true);
66+
67+
root.put("key1".getBytes(), "value1".getBytes());
68+
root.put("key2".getBytes(), "value2".getBytes());
69+
SnapshotImpl from = getSnapshotImplIns(root);
70+
from.put("key3".getBytes(), "value3".getBytes());
71+
from.put("key4".getBytes(), "value4".getBytes());
72+
73+
byte[] s1 = from.get("key1".getBytes());
74+
assertEquals(new String("value1".getBytes()), new String(s1));
75+
byte[] s2 = from.get("key2".getBytes());
76+
assertEquals(new String("value2".getBytes()), new String(s2));
77+
}
78+
79+
/**
80+
* linklist is: from2 -> from -> root
81+
* root:
82+
* from:key1=>value1, key2=>value2
83+
* from2:key3=>value3,key4=>value4
84+
* before merge: from2.mergeAhead(from);
85+
* from2: get key1 or key2, traverse 1 times
86+
* after merge
87+
* from2:key1=>value1, key2=>value2, value3=>value3,key4=>value4
88+
* from2: get key1 or key2, traverse 0 times
89+
*
90+
*/
91+
@Test
92+
public void testMergeAhead() {
93+
94+
// linklist is: from2 -> from -> root
95+
SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb());
96+
SnapshotImpl from = getSnapshotImplIns(root);
97+
from.put("key1".getBytes(), "value1".getBytes());
98+
from.put("key2".getBytes(), "value2".getBytes());
99+
100+
SnapshotImpl from2 = getSnapshotImplIns(from);
101+
from2.put("key3".getBytes(), "value3".getBytes());
102+
from2.put("key4".getBytes(), "value4".getBytes());
103+
104+
/*
105+
// before merge get data in from is success，traverse 0 times
106+
byte[] s1 = from.get("key1".getBytes());
107+
assertEquals(new String("value1".getBytes()), new String(s1));
108+
byte[] s2 = from.get("key2".getBytes());
109+
assertEquals(new String("value2".getBytes()), new String(s2));
110+
// before merge get data in from2 is success， traverse 0 times
111+
byte[] s3 = from2.get("key3".getBytes());
112+
assertEquals(new String("value3".getBytes()), new String(s3));
113+
byte[] s4 = from2.get("key4".getBytes());
114+
assertEquals(new String("value4".getBytes()), new String(s4));
115+
*/
116+
117+
// before merge from2 get data is success， traverse 1 times
118+
byte[] s11 = from2.get("key1".getBytes());
119+
assertEquals(new String("value1".getBytes()), new String(s11));
120+
byte[] s12 = from2.get("key2".getBytes());
121+
assertEquals(new String("value2".getBytes()), new String(s12));
122+
// this can not get key3 and key4
123+
assertNull(from.get("key3".getBytes()));
124+
assertNull(from.get("key4".getBytes()));
125+
126+
// do mergeAhead
127+
from2.mergeAhead(from);
128+
/*
129+
// after merge get data in from is success， traverse 0 times
130+
s1 = from.get("key1".getBytes());
131+
assertEquals(new String("value1".getBytes()), new String(s1));
132+
s2 = from.get("key2".getBytes());
133+
assertEquals(new String("value2".getBytes()), new String(s2));
134+
135+
// after merge get data in from2 is success， traverse 0 times
136+
s3 = from2.get("key3".getBytes());
137+
assertEquals(new String("value3".getBytes()), new String(s3));
138+
s4 = from2.get("key4".getBytes());
139+
assertEquals(new String("value4".getBytes()), new String(s4));
140+
*/
141+
142+
// after merge from2 get data is success， traverse 0 times
143+
byte[] s1 = from2.get("key1".getBytes());
144+
assertEquals(new String("value1".getBytes()), new String(s1));
145+
byte[] s2 = from2.get("key2".getBytes());
146+
assertEquals(new String("value2".getBytes()), new String(s2));
147+
148+
// this can not get key3 and key4
149+
assertNull(from.get("key3".getBytes()));
150+
assertNull(from.get("key4".getBytes()));
151+
}
152+
153+
/**
154+
* from: key1=>value1, key2=>value2, key3=>value31
155+
* from2: key3=>value32,key4=>value4
156+
* after merge: from2.mergeAhead(from);
157+
* from2: key1=>value1, key2=>value2, key3=>value32, key4=>value4
158+
*/
159+
@Test
160+
public void testMergeOverride() {
161+
// linklist is: from2 -> from -> root
162+
SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb());
163+
SnapshotImpl from = getSnapshotImplIns(root);
164+
from.put("key1".getBytes(), "value1".getBytes());
165+
from.put("key2".getBytes(), "value2".getBytes());
166+
from.put("key3".getBytes(), "value31".getBytes());
167+
168+
SnapshotImpl from2 = getSnapshotImplIns(from);
169+
from2.put("key3".getBytes(), "value32".getBytes());
170+
from2.put("key4".getBytes(), "value4".getBytes());
171+
// do mergeAhead
172+
from2.mergeAhead(from);
173+
174+
// after merge from2 get data is success， traverse 0 times
175+
byte[] s1 = from2.get("key1".getBytes());
176+
assertEquals(new String("value1".getBytes()), new String(s1));
177+
byte[] s2 = from2.get("key2".getBytes());
178+
assertEquals(new String("value2".getBytes()), new String(s2));
179+
byte[] s3 = from2.get("key3".getBytes());
180+
assertEquals(new String("value32".getBytes()), new String(s3));
181+
byte[] s4 = from2.get("key4".getBytes());
182+
assertEquals(new String("value4".getBytes()), new String(s4));
183+
}
184+
185+
/**
186+
* The constructor of SnapshotImpl is not public
187+
* so reflection is used to construct the object here.
188+
*/
189+
private SnapshotImpl getSnapshotImplIns(Snapshot snapshot) {
190+
Class clazz = SnapshotImpl.class;
191+
try {
192+
Constructor constructor = clazz.getDeclaredConstructor(Snapshot.class);
193+
constructor.setAccessible(true);
194+
return (SnapshotImpl) constructor.newInstance(snapshot);
195+
} catch (Exception e) {
196+
e.printStackTrace();
197+
}
198+
return null;
199+
}
200+
201+
}

0 commit comments

Comments
 (0)