diff --git a/.gitignore b/.gitignore index b632fb7..5f88ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 !.idea/ +# IA Agents stuff +.idea/copilot.* + # User-specific stuff .idea/**/workspace.xml .idea/**/tasks.xml @@ -12,6 +15,9 @@ # Generated files .idea/**/contentModel.xml .idea/php-test-framework.xml +/.idea/codeception.xml +/.idea/copilot.data.migration.agent.xml +/.idea/phpspec.xml # Sensitive or high-churn files .idea/**/dataSources/ diff --git a/docs/configuration_reference.md b/docs/configuration_reference.md index dc65eea..aafa836 100644 --- a/docs/configuration_reference.md +++ b/docs/configuration_reference.md @@ -1,329 +1,210 @@ -# Configuration Reference +# APITester Configuration Reference -## Name +This document provides a comprehensive reference for configuring APITester. The configuration is structured into **Root Configuration** and **Suite Configuration**. -```yaml -required: true -type: string -``` - -name is useful for selecting which - -## definition +## Root Configuration -## path +The top-level configuration defines global settings and the list of test suites. ```yaml -required: true -type: string +bootstrap: 'tests/bootstrap.php' # Optional: Path to a bootstrap file to include before running tests +suites: # Required: List of test suites + - ... ``` -## format +--- -```yaml -required: true -type: [ openapi ] # for now we support only the openapi format -``` +## Suite Configuration -## requester +Each suite in the `suites` list can be configured with the following options: + +### Name ```yaml -required: false -default: http-async -type: [ symfony-kernel, http-async ] +name: 'my-suite' # Required ``` +The name is used to select which suite to run via the command line (e.g., `--suite=my-suite`). -## symfony-kernel - -Forwards http requests directly to symfony kernel, has the advantage of being -compatible with transactions (in case it's -used in test cases callbacks) +### Definition -## http-async - -Sends http requests asynchronously through the network. - -## preparators +Defines the source of the API specification (e.g., OpenAPI / Swagger). ```yaml -required: false -default: all +definition: + path: 'path/to/openapi.yaml' # Required: Path or URL to the definition file + format: 'openapi' # Required: Format of the definition (currently only 'openapi' is supported) ``` -Theses are used to produce test cases, each one based on it's own logic and -configuration. -Following the list of supported preparators. +### Base URL -## examples - -## error400 +Overrides the host defined in the OpenAPI specification. ```yaml -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 400 - description: the exact statusCode to be checked +baseUrl: 'http://localhost:8080' # Optional ``` -## error401 - -```yaml -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 401 - description: the exact statusCode to be checked -``` +### Requester -## error403 +Determines how HTTP requests are executed. ```yaml -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 403 - description: the exact statusCode to be checked +requester: 'http-async' # Optional, default: 'http-async' ``` -## error404 +Available values: +- `http-async`: Sends real HTTP requests over the network asynchronously. +- `symfony-kernel`: Forwards requests directly to the Symfony Kernel (requires `symfonyKernelClass`). Useful for testing within a Symfony application context (e.g., handling transactions). -```yaml -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 404 - description: the exact statusCode to be checked -``` +#### Symfony Kernel Configuration -## error405 +If `requester` is set to `symfony-kernel`, you must specify the Kernel class. ```yaml -methods: - type: [ GET, POST, PATCH, PUT, DELETE, HEAD, OPTIONS, TRACE, CONNECT ] - default: all - description: methods to validate against -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 400 - description: the exact statusCode to be checked +symfonyKernelClass: '\App\Kernel' ``` -## error406 +### Test Case Class + +Specifies the PHPUnit TestCase class that generated tests will extend. ```yaml -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 400 - description: the exact statusCode to be checked +testCaseClass: '\OC\IntegrationTestCase' # Optional, default: '\PHPUnit\Framework\TestCase' ``` -## error413 +### PHPUnit Config + +Path to a specific PHPUnit XML configuration file for this suite. ```yaml -range: - type: array - description: describes how pagination is handled by the api -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 400 - description: the exact statusCode to be checked +phpunitConfig: 'phpunit.xml' # Optional ``` -## error416 +--- -```yaml -range: - type: array - description: describes how pagination is handled by the api -excludedFields: - type: array - description: list of response fields to be excluded when checking -schemaValidation: - type: bool - description: allows to enable or disable the schema validation. By default, the schema validation is enabled. -response: - body: - type: string - default: null - description: the exact body to be checked - statusCode: - type: int - default: 400 - description: the exact statusCode to be checked -``` +## Filters -## filters +Filters allow you to select which API operations to test based on their properties (tags, operationId, etc.). ```yaml -required: false +filters: + include: ... + exclude: ... + baseline: 'api-tester.baseline.yaml' + schemaValidationBaseline: 'api-tester.schema-baseline.yaml' ``` -## include +### Include / Exclude + +Include or exclude operations. Rules are matched against operation properties. ```yaml -required: false -type: array +include: + - { tags.*.name: 'User' } # Include operations with 'User' tag +exclude: + - { id: 'delete_user' } # Exclude operation with id 'delete_user' ``` -List of operations to include depending on the selected criteria. +### Baselines -## exclude +- `baseline`: Path to a file containing a list of failed tests to ignore in future runs. Generated via `--set-baseline` or `--update-baseline`. +- `schemaValidationBaseline`: Path to a file containing schema validation errors to ignore. -```yaml -required: false -type: array -``` +--- -List of operations to exclude depending on the selected criteria. +## Preparators -## authentication +Preparators are responsible for generating test cases for each API operation. You can configure them to customize the generated tests. ```yaml -required: false -type: array +preparators: + - name: 'error400' + # ... config ... ``` -Handles authentication, which produces a list of tokens that are +Available preparators: `examples`, `error400`, `error401`, `error403`, `error404`, `error405`, `error406`, `error413`, `error416`, `random`, `pagination_error`, `security_error`. + +### Common Configuration -## name +All preparators support the following configuration: ```yaml -required: yes -type: string +- name: 'error400' + excludedFields: ['email', 'password'] # Fields to exclude/ignore during generation + schemaValidation: false # Disable response schema validation for this preparator + headers: # Custom headers to inject + X-Custom-Header: 'value' + response: # Assertions on the response + statusCode: 400 # Expected status code (can use regex or 'NOT' tag) + body: '...' # Expected body content (regex support) + headers: # Expected response headers + Content-Type: 'application/json' ``` -## type +--- + +## Auth + +Handles authentication for the requests. You can define multiple authentication strategies. ```yaml -required: yes -type: [ oauth2_password, oauth2_implicit ] #other auth methods still need support +auth: + - name: 'admin_user' + type: 'oauth2_password' # Currently supported: oauth2_password, oauth2_implicit + # ... type specific config ... + body: + username: 'admin' + password: 'password' + grant_type: 'password' + client_id: '...' + headers: + Authorization: 'Basic ...' + filters: # Optional: Apply this auth only to specific operations + include: + - { tags.*.name: 'Admin' } ``` -## Full example +--- + +## Full Example ```yaml +bootstrap: 'tests/bootstrap.php' + suites: - - name: 'all' # name of suite, can be used to select which suite to launch + - name: 'main' definition: - path: 'src/OC/ApiBundle/Resources/schema/openclassrooms-api.yml' # path/url of the definition document - format: 'openapi' # type of the definition - requester: 'symfony-kernel' # how requests are executed, default: http-async - preparators: # are responsible of preparing test cases, leaving them empty will load all prepartors with optional config - - name: error400 # which preparator - excludedFields: [ 'body' ] - schemaValidation: false - - name: error401 - excludedFields: [ 'body' ] - - name: error403 - excludedFields: [ 'body' ] - schemaValidation: true - - name: error404 - excludedFields: [ 'body' ] - filters: # select which operations to test, filters are on operations properties - #please refer to the class APITester\Definition\Operation - include: # we include operations with the following tags - - { tags.*.name: Invitation } - - { tags.*.name: Support } - - { tags.*.name: Project } - - { tags.*.name: Course } + path: 'openapi.yaml' + format: 'openapi' + baseUrl: 'http://localhost:8000' + requester: 'symfony-kernel' + symfonyKernelClass: '\App\Kernel' + testCaseClass: '\App\Tests\ApiTestCase' + + filters: + baseline: 'tests/api-tester.baseline.yaml' + include: + - { tags.*.name: 'Public' } + - { tags.*.name: 'Private' } exclude: - # we exclude an operation with it's id for the error404 preparator - - { id: oc_api_learning_activity_course_chapter_complete_post, preparator: error404 } - - { id: oc_api_invitation_invitations_get, preparator: error401 } - - { id: oc_api_invitation_invitations_get, preparator: error403 } + - { id: 'deprecated_operation' } + + auth: + - name: 'user' + type: 'oauth2_password' + body: + username: '%env(TEST_USER)%' + password: '%env(TEST_PASSWORD)%' + grant_type: 'password' + client_id: 'app_client' + headers: + Authorization: 'Basic Y2xpZW50OnNlY3JZXdA==' - auth: # authentication requests configuration - - name: 'user_1' - body: - grant_type: 'password' - scope: 'scope 1' - username: 'tech-tests+1111111@openclassrooms.com' - password: 'test' - headers: - Authorization: 'Basic xxx' - Accept: 'application/json' - Content-Type: 'application/json' - - name: 'user_2' - body: - grant_type: 'client_credentials' - scope: 'scope2 scope3' - headers: - Authorization: 'Basic xxx' - Accept: 'application/json' - Content-Type: 'application/json' - + preparators: + - name: 'examples' + schemaValidation: true + - name: 'error400' + excludedFields: ['id', 'created_at'] + - name: 'error403' + response: + statusCode: 403 ``` diff --git a/docs/preparators-config.md b/docs/preparators-config.md new file mode 100644 index 0000000..20bc1a8 --- /dev/null +++ b/docs/preparators-config.md @@ -0,0 +1,143 @@ +# Common Configuration + +All preparators share the following common configuration options: + +```yaml +- name: 'preparator_name' + excludedFields: ['field1', 'field2'] # Optional: Fields to exclude from checks + schemaValidation: true # Optional: Enable/Disable schema validation (default: true) + headers: # Optional: Custom headers to inject in requests + X-Custom-Header: 'value' + response: # Optional: Assertions on the response + statusCode: 400 # Expected status code (int or regex) + body: '...' # Expected body content + headers: # Expected response headers + Content-Type: 'application/json' +``` + +--- + +# Specific Configuration + +## examples + +Generates test cases based on examples found in your OpenAPI definition. + +```yaml +name: examples +extensionPath: 'path/to/more/examples.yaml' # Optional: Path to external examples file +autoComplete: false # Optional: Auto-complete missing fields (default: false) +autoCreateWhenMissing: false # Optional: Create examples if missing (default: false) +``` + +## error400 + +Generates requests with invalid data to trigger 400 Bad Request. + +```yaml +name: error400 +# (Uses only common configuration) +``` + +## error401 + +Generates requests without authentication to trigger 401 Unauthorized. + +```yaml +name: error401 +# (Uses only common configuration) +``` + +## error403 + +Generates requests with insufficient permissions to trigger 403 Forbidden. + +```yaml +name: error403 +excludedTokens: ['token_name_to_skip'] # Optional: List of tokens to exclude from testing +``` + +## error404 + +Generates requests to non-existent resources/IDs to trigger 404 Not Found. + +```yaml +name: error404 +# (Uses only common configuration) +``` + +## error405 + +Generates requests with invalid HTTP methods to trigger 405 Method Not Allowed. + +```yaml +name: error405 +methods: # Optional: List of methods to test (defaults to all standard methods) + - 'GET' + - 'POST' + # ... +``` + +## error406 + +Generates requests with invalid `Accept` headers to trigger 406 Not Acceptable. + +```yaml +name: error406 +mediaTypes: [] # Optional: List of invalid media types to test +casesCount: 3 # Optional: Number of test cases to generate (default: 3) +``` + +## error413 + +Generates requests with content/parameters that are too large to trigger 413 Payload Too Large. + +```yaml +name: error413 +range: # Optional: Define ranges for pagination/size testing + - type: query # or 'header' + name: 'limit' # name of the parameter/header + lower: 0 + upper: 100 +``` + +## error416 + +Generates requests with invalid ranges to trigger 416 Range Not Satisfiable. + +```yaml +name: error416 +range: # Optional: Define ranges + - type: header + name: 'Range' + unit: 'bytes' + lower: 0 + upper: 100 +``` + +## random + +Generates requests with random valid data. + +```yaml +name: random +casesCount: 3 # Optional: Number of random variations to generate (default: 3) +``` + +## pagination_error + +Generic preparator for pagination errors. Used as base for 413/416 but can be used directly. + +```yaml +name: pagination_error +range: ... # Same as error413/error416 +``` + +## security_error + +Generic preparator for security errors. + +```yaml +name: security_error +# (Uses only common configuration) +``` \ No newline at end of file