Create tests for the updating and purging functions via the API

Signed-off-by: Benjamin Brahmer <info@b-brahmer.de>

.github/workflows/updater-test.yml

@@ -0,0 +1,80 @@
+name: Updater Tests
+on:
+  pull_request
+
+env:
+  POSTGRES_PASSWORD: nc_test_db
+  MYSQL_USER: nc_test
+  MYSQL_PASSWORD: nc_test_db
+  MYSQL_DATABASE: nc_test
+  MYSQL_PORT: 3800
+
+jobs:
+  integration:
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+    name: "Update Test: Nextcloud ${{ matrix.nextcloud }} - PHP ${{ matrix.php-versions }}"
+    strategy:
+      matrix:
+        php-versions: ['8.1']
+        nextcloud: ['stable26']
+        database: ['sqlite']
+        experimental: [false]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: ${{ matrix.php-versions }}
+          extensions: pdo_sqlite,pdo_mysql,pdo_pgsql,gd,zip
+          coverage: none
+
+      - name: Setup BATS & httpie
+        run: sudo apt-get install -y httpie && npm install -g bats@1.7.0
+
+      - name: Set up server
+        uses: SMillerDev/nextcloud-actions/setup-nextcloud@main
+        with:
+          version: ${{ matrix.nextcloud }}
+          cron: true
+          database-type: ${{ matrix.database }}
+          database-host: localhost
+          database-port: 5432
+          database-name: postgres
+          database-user: postgres
+          database-password: ${{ env.POSTGRES_PASSWORD }}
+
+      - name: Prime app build
+        run: make
+
+      - name: Configure server with app
+        uses: SMillerDev/nextcloud-actions/setup-nextcloud-app@main
+        with:
+          app: 'news'
+          check-code: false
+          force: ${{ matrix.experimental }}
+
+      - name: Install composer install php-feed-generator
+        working-directory: ../server
+        run: composer install -d apps/news/tests/test_helper/php-feed-generator
+
+      - name: Run Updater tests
+        working-directory: ../server
+        run: |
+          php -S localhost:8080 &> /tmp/webserver.log &
+          cd apps/news/tests/test_helper/feeds && php -S localhost:8090 &> /tmp/feedserver.log &
+
+          sleep 2
+
+          cd ${{ github.workspace }}/../server
+                  
+          bats apps/news/tests/updater
+
+          # Kill php server
+          kill %1
+          kill %2
+

.gitignore

@@ -14,6 +14,9 @@ site/
 
 #bats
 tests/api/helpers/settings-override.bash
+tests/test_helper/feeds/test.xml
+tests/test_helper/feeds/feed1.xml
+tests/test_helper/feeds/feed2.xml
 
 # python
 PKG-INFO

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "tests/test_helper/bats-assert"]
 	path = tests/test_helper/bats-assert
 	url = https://github.com/bats-core/bats-assert.git
+[submodule "tests/test_helper/php-feed-generator"]
+	path = tests/test_helper/php-feed-generator
+	url = https://github.com/Grotax/php-feed-generator.git

docs/admin.md

@@ -11,8 +11,11 @@ Alternatively you may use an [external updater](https://nextcloud.github.io/news
 
 Auto purging automatically removes the oldest read items of every feed after every update.
 The value you enter here is used as the limit of read items per feed, unless the feed comes with more items in it's feed.
-For example you have the default value of 200 and the feed has 210 items in it's feed.
+The individual limit per feed is only adjusted when it's bigger. Let's say last feed update came with 210 items,
-In this case the limit will be 210 instead of 200.
+then that will be the limit for that feed as long as no bigger update with more items is fetched.
+In this case the limit will be 210 instead of 200, for that feed.
+
+This is needed to prevent items from reappearing in the feed.
 
 ## Purge unread items
 This changes the behavior of the auto purging to also purge unread items. This is useful if you have users with a lot of unread items.

tests/test_helper/bats-assert

@@ -1 +1 @@
-Subproject commit ffe84ea5dd43b568851549b3e241db150c12929c
+Subproject commit 78fa631d1370562d2cd4a1390989e706158e7bf0

tests/test_helper/php-feed-generator

@@ -0,0 +1 @@
+Subproject commit 7cc16048103bf31e9a4b54293e8bbc6049a874c1

tests/updater/helpers/settings.bash

@@ -0,0 +1,8 @@
+export user=admin
+export NC_FEED="http://localhost:8090/Nextcloud.rss"
+export HEISE_FEED="http://localhost:8090/heise.xml"
+export BASE_URLv1="http://localhost:8080/index.php/apps/news/api/v1-2"
+export NC_HOST="http://localhost:8080"
+export TEST_FEED="http://localhost:8090/test.xml"
+export FEED1="http://localhost:8090/feed1.xml"
+export FEED2="http://localhost:8090/feed2.xml"

tests/updater/update.bats

@@ -0,0 +1,251 @@
+#!/usr/bin/env bats
+
+setup_file(){
+  load "helpers/settings"
+
+  if test -f "tests/api/helpers/settings-override.bash"; then
+    load "helpers/settings-override"
+  fi
+
+  export APP_PASSWORD=$(NC_PASS=${user} ./occ user:add-app-password ${user} --password-from-env | grep -Po '([A-Z|a-z|0-9]{72})')
+}
+
+teardown_file(){
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} DELETE ${NC_HOST}/ocs/v2.php/core/apppassword OCS-APIRequest:true
+}
+
+setup() {
+  load "../test_helper/bats-support/load"
+  load "../test_helper/bats-assert/load"
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 10 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+}
+
+TESTSUITE="Update"
+
+#
+# This testsuite is not intended to test the api but rather the update and purge functions.
+#
+
+teardown() {
+  # delete all feeds
+  FEED_IDS=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+  for i in $FEED_IDS; do
+    http --ignore-stdin -b -a ${user}:${APP_PASSWORD} DELETE ${BASE_URLv1}/feeds/$i > /dev/null
+  done
+
+  # delete all folders
+  FOLDER_IDS=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/folders | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+  for i in $FOLDER_IDS; do
+    http --ignore-stdin -b -a ${user}:${APP_PASSWORD} DELETE ${BASE_URLv1}/folders/$i > /dev/null
+  done
+}
+
+@test "[$TESTSUITE] Test simple update" {
+  # Create Feed
+  FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)')
+  # Get Items
+  ID_LIST1=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+  # Trigger Update
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+  # Get Items again
+  ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  assert_equal "${ID_LIST1[*]}" "${ID_LIST2[*]}"
+}
+
+@test "[$TESTSUITE] Test simple update with new content" {
+  # Create Feed
+  FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)')
+  # Get Items
+  ID_LIST1=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 15 -s 9 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+
+  # Trigger Update
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+  # Get Items again
+  ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  output="${ID_LIST2[*]}"
+
+  # Check that they are not equal but that they match partially.
+  assert_not_equal "${ID_LIST1[*]}" "${ID_LIST2[*]}"
+  assert_output --partial "${ID_LIST1[*]}"
+}
+
+@test "[$TESTSUITE] Test purge with small feed" {
+  # Generate Feed with 210 items.
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 0 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  # Create Feed
+  FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)')
+
+  # Trigger Update
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 50 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+
+  # Trigger Update
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 100 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+
+  # Trigger Update
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 150 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+
+  # Trigger Update
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 50 -s 200 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+
+  # Get Items
+  ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  # get biggest item ID
+  max=${ID_LIST[0]}
+  for n in "${ID_LIST[@]}" ; do
+      ((n > max)) && max=$n
+  done
+  
+  # mark all items of feed as read, returns nothing
+  STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)')
+  
+  # cleanup, purge items
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update
+
+  # Get unread Items, should be empty
+  output="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')"
+
+  # Get all items, also read items
+  ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  assert_equal $STATUS_CODE 200
+  # check if amount is as expected
+  assert_equal "${#ID_LIST2[@]}" 200
+
+  # unread items should be empty
+  assert_output ""
+}
+
+@test "[$TESTSUITE] Test purge with more items than default limit 200" {
+  # Generate Feed with 210 items.
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 210 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  # Create Feed
+  FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)')
+  # Get Items
+  ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  # get biggest item ID
+  max=${ID_LIST[0]}
+  for n in "${ID_LIST[@]}" ; do
+      ((n > max)) && max=$n
+  done
+  
+  # mark all items of feed as read, returns nothing
+  STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)')
+  
+  # cleanup, purge items
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update
+
+  # Get unread Items, should be empty
+  output="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')"
+
+  # Get all items, also read items
+  ID_LIST2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  assert_equal $STATUS_CODE 200
+  # check if amount is as expected
+  assert_equal "${#ID_LIST2[@]}" 210
+  assert_output ""
+}
+
+@test "[$TESTSUITE] Test Update and pruge with feed item>200; items<200" {
+  # Generate Feed with 210 items.
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 210 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  # Create Feed
+  FEEDID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$TEST_FEED | grep -Po '"id":\K([0-9]+)')
+  # Get Items
+  ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  # get biggest item ID
+  max=${ID_LIST[0]}
+  for n in "${ID_LIST[@]}" ; do
+      ((n > max)) && max=$n
+  done
+  
+  # mark all items of feed as read, returns nothing
+  STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)')
+  # cleanup, purge items
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update
+
+  FIRST_UPDATE="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')"
+
+  assert_equal "${FIRST_UPDATE}" ""
+
+  # Generate feed "update" items id 190 + 40 items = id 230
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 40 -s 190 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/test.xml
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/feeds/update userId=${user} feedId=$FEEDID
+
+  # Get Items
+  ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  # get biggest item ID
+  max=${ID_LIST[0]}
+  for n in "${ID_LIST[@]}" ; do
+      ((n > max)) && max=$n
+  done
+  
+  # mark all items of feed as read, returns nothing
+  STATUS_CODE=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEEDID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)')
+
+  SECOND_UPDATE="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')"
+
+  assert_equal "${SECOND_UPDATE}" ""
+
+  # cleanup, purge items
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update
+
+  # Get all items, also read items
+  READ_ITEMS=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  # stays at the 210 limit https://github.com/nextcloud/news/blob/ec74c1b5f3712594c7ea2139c8dfdff15d1ef826/lib/Service/FeedServiceV2.php#L287
+  assert_equal "${#READ_ITEMS[@]}" 210
+}
+
+@test "[$TESTSUITE] Test purge with two feeds with different item count limit" {
+  # Generate Feed with 260 items.
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 260 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/feed1.xml
+  # Generate Feed with 210 items.
+  php ${BATS_TEST_DIRNAME}/../test_helper/php-feed-generator/feed-generator.php -a 210 -f ${BATS_TEST_DIRNAME}/../test_helper/feeds/feed2.xml
+  
+  # Create Feeds
+  FEED1ID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$FEED1 | grep -Po '"id":\K([0-9]+)')
+  FEED2ID=$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} POST ${BASE_URLv1}/feeds url=$FEED2 | grep -Po '"id":\K([0-9]+)')
+  # Get Items
+  ID_LIST=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+  # get biggest item ID, it is enough to do this one time
+  max=${ID_LIST[0]}
+  for n in "${ID_LIST[@]}" ; do
+      ((n > max)) && max=$n
+  done
+  
+  # mark all items of both feeds as read, returns nothing
+  STATUS_CODE1=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEED1ID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)')
+  STATUS_CODE2=$(http --ignore-stdin -hdo /tmp/body -a ${user}:${APP_PASSWORD} PUT ${BASE_URLv1}/feeds/$FEED2ID/read newestItemId="$max" 2>&1| grep -Po '(?<=HTTP\/1\.1 )[0-9]{3}(?= OK)')
+  
+  # cleanup, purge items
+  http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/cleanup/after-update
+
+  # Get unread Items, should be empty
+  output="$(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items getRead=false | grep -Po '"id":\K([0-9]+)' | tr '\n' ' ')"
+
+  # Get all items of Feed Nr. 1 and 2, including read items
+  ID_LIST_FEED1=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items id="$FEED1ID" type=0 | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+  ID_LIST_FEED2=($(http --ignore-stdin -b -a ${user}:${APP_PASSWORD} GET ${BASE_URLv1}/items id="$FEED2ID" type=0 | grep -Po '"id":\K([0-9]+)' | tr '\n' ' '))
+
+
+  assert_equal $STATUS_CODE1 200
+  assert_equal $STATUS_CODE2 200
+  # check if amount is as expected
+  assert_equal "${#ID_LIST_FEED1[@]}" 260
+  assert_equal "${#ID_LIST_FEED2[@]}" 210
+  assert_output ""
+}