diff options
author | wxiaoguang <wxiaoguang@gmail.com> | 2023-05-29 10:52:32 +0800 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-05-29 10:52:32 +0800 |
commit | 84c8ab9fd109145d04d0c58cadd46160f1ee9263 (patch) | |
tree | eebfb67c7414bae68b08ba3adca2dbb1f68a61cc /modules/queue/base_levelqueue_test.go | |
parent | 8faf9465b3913137d3470151a678857c319625a5 (diff) | |
download | gitea-84c8ab9fd109145d04d0c58cadd46160f1ee9263.tar.gz gitea-84c8ab9fd109145d04d0c58cadd46160f1ee9263.zip |
Help to recover from corrupted levelqueue (#24912)
gitea.com experienced the corrupted LevelQueue bug again.
I think the problem is clear now: if the keys in LevelDB went
out-of-sync, the LevelQueue itself doesn't have the ability to recover,
eg:
* LevelQueue.Len() reports 100
* LevelQueue.LPop() reports ErrNotFound = errors.New("no key found")
So it needs to dive into the LevelDB to remove all keys to recover the
corrupted LevelQueue.
More comments are in TestCorruptedLevelQueue.
Diffstat (limited to 'modules/queue/base_levelqueue_test.go')
-rw-r--r-- | modules/queue/base_levelqueue_test.go | 55 |
1 files changed, 55 insertions, 0 deletions
diff --git a/modules/queue/base_levelqueue_test.go b/modules/queue/base_levelqueue_test.go index 712a0892cd..b881802ca2 100644 --- a/modules/queue/base_levelqueue_test.go +++ b/modules/queue/base_levelqueue_test.go @@ -6,9 +6,12 @@ package queue import ( "testing" + "code.gitea.io/gitea/modules/queue/lqinternal" "code.gitea.io/gitea/modules/setting" + "gitea.com/lunny/levelqueue" "github.com/stretchr/testify/assert" + "github.com/syndtr/goleveldb/leveldb" ) func TestBaseLevelDB(t *testing.T) { @@ -21,3 +24,55 @@ func TestBaseLevelDB(t *testing.T) { testQueueBasic(t, newBaseLevelQueueSimple, toBaseConfig("baseLevelQueue", setting.QueueSettings{Datadir: t.TempDir() + "/queue-test", Length: 10}), false) testQueueBasic(t, newBaseLevelQueueUnique, toBaseConfig("baseLevelQueueUnique", setting.QueueSettings{ConnStr: "leveldb://" + t.TempDir() + "/queue-test", Length: 10}), true) } + +func TestCorruptedLevelQueue(t *testing.T) { + // sometimes the levelqueue could be in a corrupted state, this test is to make sure it can recover from it + dbDir := t.TempDir() + "/levelqueue-test" + db, err := leveldb.OpenFile(dbDir, nil) + if !assert.NoError(t, err) { + return + } + defer db.Close() + + assert.NoError(t, db.Put([]byte("other-key"), []byte("other-value"), nil)) + + nameQueuePrefix := []byte("queue_name") + nameSetPrefix := []byte("set_name") + lq, err := levelqueue.NewUniqueQueue(db, nameQueuePrefix, nameSetPrefix, false) + assert.NoError(t, err) + assert.NoError(t, lq.RPush([]byte("item-1"))) + + itemKey := lqinternal.QueueItemKeyBytes(nameQueuePrefix, 1) + itemValue, err := db.Get(itemKey, nil) + assert.NoError(t, err) + assert.Equal(t, []byte("item-1"), itemValue) + + // there should be 5 keys in db: queue low, queue high, 1 queue item, 1 set item, and "other-key" + keys := lqinternal.ListLevelQueueKeys(db) + assert.Len(t, keys, 5) + + // delete the queue item key, to corrupt the queue + assert.NoError(t, db.Delete(itemKey, nil)) + // now the queue is corrupted, it never works again + _, err = lq.LPop() + assert.ErrorIs(t, err, levelqueue.ErrNotFound) + assert.NoError(t, lq.Close()) + + // remove all the queue related keys to reset the queue + lqinternal.RemoveLevelQueueKeys(db, nameQueuePrefix) + lqinternal.RemoveLevelQueueKeys(db, nameSetPrefix) + // now there should be only 1 key in db: "other-key" + keys = lqinternal.ListLevelQueueKeys(db) + assert.Len(t, keys, 1) + assert.Equal(t, []byte("other-key"), keys[0]) + + // re-create a queue from db + lq, err = levelqueue.NewUniqueQueue(db, nameQueuePrefix, nameSetPrefix, false) + assert.NoError(t, err) + assert.NoError(t, lq.RPush([]byte("item-new-1"))) + // now the queue works again + itemValue, err = lq.LPop() + assert.NoError(t, err) + assert.Equal(t, []byte("item-new-1"), itemValue) + assert.NoError(t, lq.Close()) +} |